[yt-dlp.git] / yt_dlp / extractor / radiko.py

import base64
import re
import urllib.parse

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    clean_html,
    time_seconds,
    try_call,
    unified_timestamp,
    update_url_query,
)


class RadikoBaseIE(InfoExtractor):
    _FULL_KEY = None

    def _auth_client(self):
        _, auth1_handle = self._download_webpage_handle(
            'https://radiko.jp/v2/api/auth1', None, 'Downloading authentication page',
            headers={
                'x-radiko-app': 'pc_html5',
                'x-radiko-app-version': '0.0.1',
                'x-radiko-device': 'pc',
                'x-radiko-user': 'dummy_user',
            })
        auth1_header = auth1_handle.info()

        auth_token = auth1_header['X-Radiko-AuthToken']
        kl = int(auth1_header['X-Radiko-KeyLength'])
        ko = int(auth1_header['X-Radiko-KeyOffset'])
        raw_partial_key = self._extract_full_key()[ko:ko + kl]
        partial_key = base64.b64encode(raw_partial_key).decode()

        area_id = self._download_webpage(
            'https://radiko.jp/v2/api/auth2', None, 'Authenticating',
            headers={
                'x-radiko-device': 'pc',
                'x-radiko-user': 'dummy_user',
                'x-radiko-authtoken': auth_token,
                'x-radiko-partialkey': partial_key,
            }).split(',')[0]

        auth_data = (auth_token, area_id)
        self.cache.store('radiko', 'auth_data', auth_data)
        return auth_data

    def _extract_full_key(self):
        if self._FULL_KEY:
            return self._FULL_KEY

        jscode = self._download_webpage(
            'https://radiko.jp/apps/js/playerCommon.js', None,
            note='Downloading player js code')
        full_key = self._search_regex(
            (r"RadikoJSPlayer\([^,]*,\s*(['\"])pc_html5\1,\s*(['\"])(?P<fullkey>[0-9a-f]+)\2,\s*{"),
            jscode, 'full key', fatal=False, group='fullkey')

        if full_key:
            full_key = full_key.encode()
        else:  # use full key ever known
            full_key = b'bcd151073c03b352e1ef2fd66c32209da9ca0afa'

        self._FULL_KEY = full_key
        return full_key

    def _find_program(self, video_id, station, cursor):
        station_program = self._download_xml(
            'https://radiko.jp/v3/program/station/weekly/%s.xml' % station, video_id,
            note='Downloading radio program for %s station' % station)

        prog = None
        for p in station_program.findall('.//prog'):
            ft_str, to_str = p.attrib['ft'], p.attrib['to']
            ft = unified_timestamp(ft_str, False)
            to = unified_timestamp(to_str, False)
            if ft <= cursor and cursor < to:
                prog = p
                break
        if not prog:
            raise ExtractorError('Cannot identify radio program to download!')
        assert ft, to
        return prog, station_program, ft, ft_str, to_str

    def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token, area_id, query):
        m3u8_playlist_data = self._download_xml(
            f'https://radiko.jp/v3/station/stream/pc_html5/{station}.xml', video_id,
            note='Downloading stream information')
        m3u8_urls = m3u8_playlist_data.findall('.//url')

        formats = []
        found = set()
        for url_tag in m3u8_urls:
            pcu = url_tag.find('playlist_create_url')
            url_attrib = url_tag.attrib
            playlist_url = update_url_query(pcu.text, {
                'station_id': station,
                **query,
                'l': '15',
                'lsid': '88ecea37e968c1f17d5413312d9f8003',
                'type': 'b',
            })
            if playlist_url in found:
                continue
            else:
                found.add(playlist_url)

            time_to_skip = None if is_onair else cursor - ft

            domain = urllib.parse.urlparse(playlist_url).netloc
            subformats = self._extract_m3u8_formats(
                playlist_url, video_id, ext='m4a',
                live=True, fatal=False, m3u8_id=domain,
                note=f'Downloading m3u8 information from {domain}',
                headers={
                    'X-Radiko-AreaId': area_id,
                    'X-Radiko-AuthToken': auth_token,
                })
            for sf in subformats:
                if re.fullmatch(r'[cf]-radiko\.smartstream\.ne\.jp', domain):
                    # Prioritize live radio vs playback based on extractor
                    sf['preference'] = 100 if is_onair else -100
                if not is_onair and url_attrib['timefree'] == '1' and time_to_skip:
                    sf['downloader_options'] = {'ffmpeg_args': ['-ss', time_to_skip]}
            formats.extend(subformats)

        return formats


class RadikoIE(RadikoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<id>\d+)'

    _TESTS = [{
        # QRR (文化放送) station provides <desc>
        'url': 'https://radiko.jp/#!/ts/QRR/20210425101300',
        'only_matching': True,
    }, {
        # FMT (TOKYO FM) station does not provide <desc>
        'url': 'https://radiko.jp/#!/ts/FMT/20210810150000',
        'only_matching': True,
    }, {
        'url': 'https://radiko.jp/#!/ts/JOAK-FM/20210509090000',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        station, video_id = self._match_valid_url(url).groups()
        vid_int = unified_timestamp(video_id, False)
        prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int)

        auth_cache = self.cache.load('radiko', 'auth_data')
        for attempt in range(2):
            auth_token, area_id = (not attempt and auth_cache) or self._auth_client()
            formats = self._extract_formats(
                video_id=video_id, station=station, is_onair=False,
                ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id,
                query={
                    'start_at': radio_begin,
                    'ft': radio_begin,
                    'end_at': radio_end,
                    'to': radio_end,
                    'seek': video_id,
                })
            if formats:
                break

        return {
            'id': video_id,
            'title': try_call(lambda: prog.find('title').text),
            'description': clean_html(try_call(lambda: prog.find('info').text)),
            'uploader': try_call(lambda: station_program.find('.//name').text),
            'uploader_id': station,
            'timestamp': vid_int,
            'formats': formats,
            'is_live': True,
        }


class RadikoRadioIE(RadikoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/live/(?P<id>[A-Z0-9-]+)'

    _TESTS = [{
        # QRR (文化放送) station provides <desc>
        'url': 'https://radiko.jp/#!/live/QRR',
        'only_matching': True,
    }, {
        # FMT (TOKYO FM) station does not provide <desc>
        'url': 'https://radiko.jp/#!/live/FMT',
        'only_matching': True,
    }, {
        'url': 'https://radiko.jp/#!/live/JOAK-FM',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        station = self._match_id(url)
        self.report_warning('Downloader will not stop at the end of the program! Press Ctrl+C to stop')

        auth_token, area_id = self._auth_client()
        # get current time in JST (GMT+9:00 w/o DST)
        vid_now = time_seconds(hours=9)

        prog, station_program, ft, _, _ = self._find_program(station, station, vid_now)

        title = prog.find('title').text
        description = clean_html(prog.find('info').text)
        station_name = station_program.find('.//name').text

        formats = self._extract_formats(
            video_id=station, station=station, is_onair=True,
            ft=ft, cursor=vid_now, auth_token=auth_token, area_id=area_id,
            query={})

        return {
            'id': station,
            'title': title,
            'description': description,
            'uploader': station_name,
            'uploader_id': station,
            'timestamp': ft,
            'formats': formats,
            'is_live': True,
        }
Commit	Line	Data
1931a55e	1	import base64
1f8b4ab7 L	2	import re
1f8b4ab7 L	3	import urllib.parse
1931a55e THD	4
	5	from .common import InfoExtractor
	6	from ..utils import (
	7	ExtractorError,
1931a55e	8	clean_html,
1f8b4ab7 L	9	time_seconds,
1f8b4ab7 L	10	try_call,
1931a55e	11	unified_timestamp,
1f8b4ab7	12	update_url_query,
1931a55e	13	)
1931a55e THD	14
	15
	16	class RadikoBaseIE(InfoExtractor):
	17	_FULL_KEY = None
	18
	19	def _auth_client(self):
1931a55e THD	20	_, auth1_handle = self._download_webpage_handle(
	21	'https://radiko.jp/v2/api/auth1', None, 'Downloading authentication page',
	22	headers={
	23	'x-radiko-app': 'pc_html5',
	24	'x-radiko-app-version': '0.0.1',
	25	'x-radiko-device': 'pc',
	26	'x-radiko-user': 'dummy_user',
	27	})
	28	auth1_header = auth1_handle.info()
	29
	30	auth_token = auth1_header['X-Radiko-AuthToken']
	31	kl = int(auth1_header['X-Radiko-KeyLength'])
	32	ko = int(auth1_header['X-Radiko-KeyOffset'])
	33	raw_partial_key = self._extract_full_key()[ko:ko + kl]
	34	partial_key = base64.b64encode(raw_partial_key).decode()
	35
	36	area_id = self._download_webpage(
	37	'https://radiko.jp/v2/api/auth2', None, 'Authenticating',
	38	headers={
	39	'x-radiko-device': 'pc',
	40	'x-radiko-user': 'dummy_user',
	41	'x-radiko-authtoken': auth_token,
	42	'x-radiko-partialkey': partial_key,
	43	}).split(',')[0]
	44
	45	auth_data = (auth_token, area_id)
9809740b	46	self.cache.store('radiko', 'auth_data', auth_data)
1931a55e THD	47	return auth_data
	48
	49	def _extract_full_key(self):
	50	if self._FULL_KEY:
	51	return self._FULL_KEY
	52
	53	jscode = self._download_webpage(
	54	'https://radiko.jp/apps/js/playerCommon.js', None,
	55	note='Downloading player js code')
	56	full_key = self._search_regex(
	57	(r"RadikoJSPlayer\([^,],\s(['\"])pc_html5\1,\s(['\"])(?P<fullkey>[0-9a-f]+)\2,\s{"),
	58	jscode, 'full key', fatal=False, group='fullkey')
	59
	60	if full_key:
	61	full_key = full_key.encode()
	62	else: # use full key ever known
	63	full_key = b'bcd151073c03b352e1ef2fd66c32209da9ca0afa'
	64
	65	self._FULL_KEY = full_key
	66	return full_key
	67
	68	def _find_program(self, video_id, station, cursor):
	69	station_program = self._download_xml(
	70	'https://radiko.jp/v3/program/station/weekly/%s.xml' % station, video_id,
	71	note='Downloading radio program for %s station' % station)
	72
	73	prog = None
	74	for p in station_program.findall('.//prog'):
	75	ft_str, to_str = p.attrib['ft'], p.attrib['to']
	76	ft = unified_timestamp(ft_str, False)
	77	to = unified_timestamp(to_str, False)
	78	if ft <= cursor and cursor < to:
	79	prog = p
	80	break
	81	if not prog:
	82	raise ExtractorError('Cannot identify radio program to download!')
	83	assert ft, to
	84	return prog, station_program, ft, ft_str, to_str
	85
	86	def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token, area_id, query):
	87	m3u8_playlist_data = self._download_xml(
1f8b4ab7 L	88	f'https://radiko.jp/v3/station/stream/pc_html5/{station}.xml', video_id,
1f8b4ab7 L	89	note='Downloading stream information')
1931a55e THD	90	m3u8_urls = m3u8_playlist_data.findall('.//url')
	91
	92	formats = []
	93	found = set()
	94	for url_tag in m3u8_urls:
	95	pcu = url_tag.find('playlist_create_url')
	96	url_attrib = url_tag.attrib
	97	playlist_url = update_url_query(pcu.text, {
	98	'station_id': station,
	99	**query,
	100	'l': '15',
1f8b4ab7	101	'lsid': '88ecea37e968c1f17d5413312d9f8003',
1931a55e THD	102	'type': 'b',
	103	})
	104	if playlist_url in found:
	105	continue
	106	else:
	107	found.add(playlist_url)
	108
	109	time_to_skip = None if is_onair else cursor - ft
	110
1f8b4ab7	111	domain = urllib.parse.urlparse(playlist_url).netloc
1931a55e THD	112	subformats = self._extract_m3u8_formats(
1931a55e THD	113	playlist_url, video_id, ext='m4a',
1f8b4ab7 L	114	live=True, fatal=False, m3u8_id=domain,
1f8b4ab7 L	115	note=f'Downloading m3u8 information from {domain}',
1931a55e THD	116	headers={
	117	'X-Radiko-AreaId': area_id,
	118	'X-Radiko-AuthToken': auth_token,
	119	})
	120	for sf in subformats:
1f8b4ab7	121	if re.fullmatch(r'[cf]-radiko\.smartstream\.ne\.jp', domain):
1931a55e THD	122	# Prioritize live radio vs playback based on extractor
	123	sf['preference'] = 100 if is_onair else -100
	124	if not is_onair and url_attrib['timefree'] == '1' and time_to_skip:
0a5a191a	125	sf['downloader_options'] = {'ffmpeg_args': ['-ss', time_to_skip]}
1931a55e THD	126	formats.extend(subformats)
1931a55e THD	127
1931a55e THD	128	return formats
	129
	130
	131	class RadikoIE(RadikoBaseIE):
	132	_VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<id>\d+)'
	133
	134	_TESTS = [{
	135	# QRR (文化放送) station provides <desc>
	136	'url': 'https://radiko.jp/#!/ts/QRR/20210425101300',
	137	'only_matching': True,
	138	}, {
	139	# FMT (TOKYO FM) station does not provide <desc>
	140	'url': 'https://radiko.jp/#!/ts/FMT/20210810150000',
	141	'only_matching': True,
	142	}, {
	143	'url': 'https://radiko.jp/#!/ts/JOAK-FM/20210509090000',
	144	'only_matching': True,
	145	}]
	146
	147	def _real_extract(self, url):
	148	station, video_id = self._match_valid_url(url).groups()
	149	vid_int = unified_timestamp(video_id, False)
1931a55e THD	150	prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int)
1931a55e THD	151
9809740b	152	auth_cache = self.cache.load('radiko', 'auth_data')
1f8b4ab7 L	153	for attempt in range(2):
	154	auth_token, area_id = (not attempt and auth_cache) or self._auth_client()
	155	formats = self._extract_formats(
	156	video_id=video_id, station=station, is_onair=False,
	157	ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id,
	158	query={
	159	'start_at': radio_begin,
	160	'ft': radio_begin,
	161	'end_at': radio_end,
	162	'to': radio_end,
	163	'seek': video_id,
	164	})
	165	if formats:
	166	break
1931a55e THD	167
	168	return {
	169	'id': video_id,
1f8b4ab7 L	170	'title': try_call(lambda: prog.find('title').text),
	171	'description': clean_html(try_call(lambda: prog.find('info').text)),
	172	'uploader': try_call(lambda: station_program.find('.//name').text),
1931a55e THD	173	'uploader_id': station,
	174	'timestamp': vid_int,
	175	'formats': formats,
	176	'is_live': True,
	177	}
	178
	179
	180	class RadikoRadioIE(RadikoBaseIE):
	181	_VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/live/(?P<id>[A-Z0-9-]+)'
	182
	183	_TESTS = [{
	184	# QRR (文化放送) station provides <desc>
	185	'url': 'https://radiko.jp/#!/live/QRR',
	186	'only_matching': True,
	187	}, {
	188	# FMT (TOKYO FM) station does not provide <desc>
	189	'url': 'https://radiko.jp/#!/live/FMT',
	190	'only_matching': True,
	191	}, {
	192	'url': 'https://radiko.jp/#!/live/JOAK-FM',
	193	'only_matching': True,
	194	}]
	195
	196	def _real_extract(self, url):
	197	station = self._match_id(url)
	198	self.report_warning('Downloader will not stop at the end of the program! Press Ctrl+C to stop')
	199
	200	auth_token, area_id = self._auth_client()
	201	# get current time in JST (GMT+9:00 w/o DST)
1f8b4ab7	202	vid_now = time_seconds(hours=9)
1931a55e THD	203
	204	prog, station_program, ft, _, _ = self._find_program(station, station, vid_now)
	205
	206	title = prog.find('title').text
	207	description = clean_html(prog.find('info').text)
	208	station_name = station_program.find('.//name').text
	209
	210	formats = self._extract_formats(
	211	video_id=station, station=station, is_onair=True,
	212	ft=ft, cursor=vid_now, auth_token=auth_token, area_id=area_id,
	213	query={})
	214
	215	return {
	216	'id': station,
	217	'title': title,
	218	'description': description,
	219	'uploader': station_name,
	220	'uploader_id': station,
	221	'timestamp': ft,
	222	'formats': formats,
	223	'is_live': True,
	224	}