[yt-dlp.git] / yt_dlp / extractor / radiko.py

# coding: utf-8
from __future__ import unicode_literals

import re
import base64
import calendar
import datetime

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    update_url_query,
    clean_html,
    unified_timestamp,
)
from ..compat import compat_urllib_parse


class RadikoBaseIE(InfoExtractor):
    _FULL_KEY = None

    def _auth_client(self):
        auth_cache = self._downloader.cache.load('radiko', 'auth_data')
        if auth_cache:
            return auth_cache

        _, auth1_handle = self._download_webpage_handle(
            'https://radiko.jp/v2/api/auth1', None, 'Downloading authentication page',
            headers={
                'x-radiko-app': 'pc_html5',
                'x-radiko-app-version': '0.0.1',
                'x-radiko-device': 'pc',
                'x-radiko-user': 'dummy_user',
            })
        auth1_header = auth1_handle.info()

        auth_token = auth1_header['X-Radiko-AuthToken']
        kl = int(auth1_header['X-Radiko-KeyLength'])
        ko = int(auth1_header['X-Radiko-KeyOffset'])
        raw_partial_key = self._extract_full_key()[ko:ko + kl]
        partial_key = base64.b64encode(raw_partial_key).decode()

        area_id = self._download_webpage(
            'https://radiko.jp/v2/api/auth2', None, 'Authenticating',
            headers={
                'x-radiko-device': 'pc',
                'x-radiko-user': 'dummy_user',
                'x-radiko-authtoken': auth_token,
                'x-radiko-partialkey': partial_key,
            }).split(',')[0]

        auth_data = (auth_token, area_id)
        self._downloader.cache.store('radiko', 'auth_data', auth_data)
        return auth_data

    def _extract_full_key(self):
        if self._FULL_KEY:
            return self._FULL_KEY

        jscode = self._download_webpage(
            'https://radiko.jp/apps/js/playerCommon.js', None,
            note='Downloading player js code')
        full_key = self._search_regex(
            (r"RadikoJSPlayer\([^,]*,\s*(['\"])pc_html5\1,\s*(['\"])(?P<fullkey>[0-9a-f]+)\2,\s*{"),
            jscode, 'full key', fatal=False, group='fullkey')

        if full_key:
            full_key = full_key.encode()
        else:  # use full key ever known
            full_key = b'bcd151073c03b352e1ef2fd66c32209da9ca0afa'

        self._FULL_KEY = full_key
        return full_key

    def _find_program(self, video_id, station, cursor):
        station_program = self._download_xml(
            'https://radiko.jp/v3/program/station/weekly/%s.xml' % station, video_id,
            note='Downloading radio program for %s station' % station)

        prog = None
        for p in station_program.findall('.//prog'):
            ft_str, to_str = p.attrib['ft'], p.attrib['to']
            ft = unified_timestamp(ft_str, False)
            to = unified_timestamp(to_str, False)
            if ft <= cursor and cursor < to:
                prog = p
                break
        if not prog:
            raise ExtractorError('Cannot identify radio program to download!')
        assert ft, to
        return prog, station_program, ft, ft_str, to_str

    def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token, area_id, query):
        m3u8_playlist_data = self._download_xml(
            'https://radiko.jp/v3/station/stream/pc_html5/%s.xml' % station, video_id,
            note='Downloading m3u8 information')
        m3u8_urls = m3u8_playlist_data.findall('.//url')

        formats = []
        found = set()
        for url_tag in m3u8_urls:
            pcu = url_tag.find('playlist_create_url')
            url_attrib = url_tag.attrib
            playlist_url = update_url_query(pcu.text, {
                'station_id': station,
                **query,
                'l': '15',
                'lsid': '77d0678df93a1034659c14d6fc89f018',
                'type': 'b',
            })
            if playlist_url in found:
                continue
            else:
                found.add(playlist_url)

            time_to_skip = None if is_onair else cursor - ft

            subformats = self._extract_m3u8_formats(
                playlist_url, video_id, ext='m4a',
                live=True, fatal=False, m3u8_id=None,
                headers={
                    'X-Radiko-AreaId': area_id,
                    'X-Radiko-AuthToken': auth_token,
                })
            for sf in subformats:
                domain = sf['format_id'] = compat_urllib_parse.urlparse(sf['url']).netloc
                if re.match(r'^[cf]-radiko\.smartstream\.ne\.jp$', domain):
                    # Prioritize live radio vs playback based on extractor
                    sf['preference'] = 100 if is_onair else -100
                if not is_onair and url_attrib['timefree'] == '1' and time_to_skip:
                    sf['_ffmpeg_args'] = ['-ss', time_to_skip]
            formats.extend(subformats)

        self._sort_formats(formats)
        return formats


class RadikoIE(RadikoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<id>\d+)'

    _TESTS = [{
        # QRR (文化放送) station provides <desc>
        'url': 'https://radiko.jp/#!/ts/QRR/20210425101300',
        'only_matching': True,
    }, {
        # FMT (TOKYO FM) station does not provide <desc>
        'url': 'https://radiko.jp/#!/ts/FMT/20210810150000',
        'only_matching': True,
    }, {
        'url': 'https://radiko.jp/#!/ts/JOAK-FM/20210509090000',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        station, video_id = self._match_valid_url(url).groups()
        vid_int = unified_timestamp(video_id, False)

        auth_token, area_id = self._auth_client()

        prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int)

        title = prog.find('title').text
        description = clean_html(prog.find('info').text)
        station_name = station_program.find('.//name').text

        formats = self._extract_formats(
            video_id=video_id, station=station, is_onair=False,
            ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id,
            query={
                'start_at': radio_begin,
                'ft': radio_begin,
                'end_at': radio_end,
                'to': radio_end,
                'seek': video_id,
            })

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'uploader': station_name,
            'uploader_id': station,
            'timestamp': vid_int,
            'formats': formats,
            'is_live': True,
        }


class RadikoRadioIE(RadikoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/live/(?P<id>[A-Z0-9-]+)'

    _TESTS = [{
        # QRR (文化放送) station provides <desc>
        'url': 'https://radiko.jp/#!/live/QRR',
        'only_matching': True,
    }, {
        # FMT (TOKYO FM) station does not provide <desc>
        'url': 'https://radiko.jp/#!/live/FMT',
        'only_matching': True,
    }, {
        'url': 'https://radiko.jp/#!/live/JOAK-FM',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        station = self._match_id(url)
        self.report_warning('Downloader will not stop at the end of the program! Press Ctrl+C to stop')

        auth_token, area_id = self._auth_client()
        # get current time in JST (GMT+9:00 w/o DST)
        vid_now = datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=9)))
        vid_now = calendar.timegm(vid_now.timetuple())

        prog, station_program, ft, _, _ = self._find_program(station, station, vid_now)

        title = prog.find('title').text
        description = clean_html(prog.find('info').text)
        station_name = station_program.find('.//name').text

        formats = self._extract_formats(
            video_id=station, station=station, is_onair=True,
            ft=ft, cursor=vid_now, auth_token=auth_token, area_id=area_id,
            query={})

        return {
            'id': station,
            'title': title,
            'description': description,
            'uploader': station_name,
            'uploader_id': station,
            'timestamp': ft,
            'formats': formats,
            'is_live': True,
        }
Commit	Line	Data
1931a55e THD	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
	4	import re
	5	import base64
	6	import calendar
	7	import datetime
	8
	9	from .common import InfoExtractor
	10	from ..utils import (
	11	ExtractorError,
	12	update_url_query,
	13	clean_html,
	14	unified_timestamp,
	15	)
	16	from ..compat import compat_urllib_parse
	17
	18
	19	class RadikoBaseIE(InfoExtractor):
	20	_FULL_KEY = None
	21
	22	def _auth_client(self):
	23	auth_cache = self._downloader.cache.load('radiko', 'auth_data')
	24	if auth_cache:
	25	return auth_cache
	26
	27	_, auth1_handle = self._download_webpage_handle(
	28	'https://radiko.jp/v2/api/auth1', None, 'Downloading authentication page',
	29	headers={
	30	'x-radiko-app': 'pc_html5',
	31	'x-radiko-app-version': '0.0.1',
	32	'x-radiko-device': 'pc',
	33	'x-radiko-user': 'dummy_user',
	34	})
	35	auth1_header = auth1_handle.info()
	36
	37	auth_token = auth1_header['X-Radiko-AuthToken']
	38	kl = int(auth1_header['X-Radiko-KeyLength'])
	39	ko = int(auth1_header['X-Radiko-KeyOffset'])
	40	raw_partial_key = self._extract_full_key()[ko:ko + kl]
	41	partial_key = base64.b64encode(raw_partial_key).decode()
	42
	43	area_id = self._download_webpage(
	44	'https://radiko.jp/v2/api/auth2', None, 'Authenticating',
	45	headers={
	46	'x-radiko-device': 'pc',
	47	'x-radiko-user': 'dummy_user',
	48	'x-radiko-authtoken': auth_token,
	49	'x-radiko-partialkey': partial_key,
	50	}).split(',')[0]
	51
	52	auth_data = (auth_token, area_id)
	53	self._downloader.cache.store('radiko', 'auth_data', auth_data)
	54	return auth_data
	55
	56	def _extract_full_key(self):
	57	if self._FULL_KEY:
	58	return self._FULL_KEY
	59
	60	jscode = self._download_webpage(
	61	'https://radiko.jp/apps/js/playerCommon.js', None,
	62	note='Downloading player js code')
	63	full_key = self._search_regex(
	64	(r"RadikoJSPlayer\([^,],\s(['\"])pc_html5\1,\s(['\"])(?P<fullkey>[0-9a-f]+)\2,\s{"),
65	jscode, 'full key', fatal=False, group='fullkey')
66
67	if full_key:
68	full_key = full_key.encode()
69	else: # use full key ever known
70	full_key = b'bcd151073c03b352e1ef2fd66c32209da9ca0afa'
71
72	self._FULL_KEY = full_key
73	return full_key
74
75	def _find_program(self, video_id, station, cursor):
76	station_program = self._download_xml(
77	'https://radiko.jp/v3/program/station/weekly/%s.xml' % station, video_id,
78	note='Downloading radio program for %s station' % station)
79
80	prog = None
81	for p in station_program.findall('.//prog'):
82	ft_str, to_str = p.attrib['ft'], p.attrib['to']
83	ft = unified_timestamp(ft_str, False)
84	to = unified_timestamp(to_str, False)
85	if ft <= cursor and cursor < to:
86	prog = p
87	break
88	if not prog:
89	raise ExtractorError('Cannot identify radio program to download!')
90	assert ft, to
91	return prog, station_program, ft, ft_str, to_str
92
93	def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token, area_id, query):
94	m3u8_playlist_data = self._download_xml(
95	'https://radiko.jp/v3/station/stream/pc_html5/%s.xml' % station, video_id,
96	note='Downloading m3u8 information')
97	m3u8_urls = m3u8_playlist_data.findall('.//url')
98
99	formats = []
100	found = set()
101	for url_tag in m3u8_urls:
102	pcu = url_tag.find('playlist_create_url')
103	url_attrib = url_tag.attrib
104	playlist_url = update_url_query(pcu.text, {
105	'station_id': station,
106	**query,
107	'l': '15',
108	'lsid': '77d0678df93a1034659c14d6fc89f018',
109	'type': 'b',
110	})
111	if playlist_url in found:
112	continue
113	else:
114	found.add(playlist_url)
115
116	time_to_skip = None if is_onair else cursor - ft
117
118	subformats = self._extract_m3u8_formats(
119	playlist_url, video_id, ext='m4a',
120	live=True, fatal=False, m3u8_id=None,
121	headers={
122	'X-Radiko-AreaId': area_id,
123	'X-Radiko-AuthToken': auth_token,
124	})
125	for sf in subformats:
126	domain = sf['format_id'] = compat_urllib_parse.urlparse(sf['url']).netloc
127	if re.match(r'^[cf]-radiko\.smartstream\.ne\.jp$', domain):
128	# Prioritize live radio vs playback based on extractor
129	sf['preference'] = 100 if is_onair else -100
130	if not is_onair and url_attrib['timefree'] == '1' and time_to_skip:
131	sf['_ffmpeg_args'] = ['-ss', time_to_skip]
132	formats.extend(subformats)
133
134	self._sort_formats(formats)
135	return formats
136
137
138	class RadikoIE(RadikoBaseIE):
139	_VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<id>\d+)'
140
141	_TESTS = [{
142	# QRR (文化放送) station provides <desc>
143	'url': 'https://radiko.jp/#!/ts/QRR/20210425101300',
144	'only_matching': True,
145	}, {
146	# FMT (TOKYO FM) station does not provide <desc>
147	'url': 'https://radiko.jp/#!/ts/FMT/20210810150000',
148	'only_matching': True,
149	}, {
150	'url': 'https://radiko.jp/#!/ts/JOAK-FM/20210509090000',
151	'only_matching': True,
152	}]
153
154	def _real_extract(self, url):
155	station, video_id = self._match_valid_url(url).groups()
156	vid_int = unified_timestamp(video_id, False)
157
158	auth_token, area_id = self._auth_client()
159
160	prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int)
161
162	title = prog.find('title').text
163	description = clean_html(prog.find('info').text)
164	station_name = station_program.find('.//name').text
165
166	formats = self._extract_formats(
167	video_id=video_id, station=station, is_onair=False,
168	ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id,
169	query={
170	'start_at': radio_begin,
171	'ft': radio_begin,
172	'end_at': radio_end,
173	'to': radio_end,
174	'seek': video_id,
175	})
176
177	return {
178	'id': video_id,
179	'title': title,
180	'description': description,
181	'uploader': station_name,
182	'uploader_id': station,
183	'timestamp': vid_int,
184	'formats': formats,
185	'is_live': True,
186	}
187
188
189	class RadikoRadioIE(RadikoBaseIE):
190	_VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/live/(?P<id>[A-Z0-9-]+)'
191
192	_TESTS = [{
193	# QRR (文化放送) station provides <desc>
194	'url': 'https://radiko.jp/#!/live/QRR',
195	'only_matching': True,
196	}, {
197	# FMT (TOKYO FM) station does not provide <desc>
198	'url': 'https://radiko.jp/#!/live/FMT',
199	'only_matching': True,
200	}, {
201	'url': 'https://radiko.jp/#!/live/JOAK-FM',
202	'only_matching': True,
203	}]
204
205	def _real_extract(self, url):
206	station = self._match_id(url)
207	self.report_warning('Downloader will not stop at the end of the program! Press Ctrl+C to stop')
208
209	auth_token, area_id = self._auth_client()
210	# get current time in JST (GMT+9:00 w/o DST)
211	vid_now = datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=9)))
212	vid_now = calendar.timegm(vid_now.timetuple())
213
214	prog, station_program, ft, _, _ = self._find_program(station, station, vid_now)
215
216	title = prog.find('title').text
217	description = clean_html(prog.find('info').text)
218	station_name = station_program.find('.//name').text
219
220	formats = self._extract_formats(
221	video_id=station, station=station, is_onair=True,
222	ft=ft, cursor=vid_now, auth_token=auth_token, area_id=area_id,
223	query={})
224
225	return {
226	'id': station,
227	'title': title,
228	'description': description,
229	'uploader': station_name,
230	'uploader_id': station,
231	'timestamp': ft,
232	'formats': formats,
233	'is_live': True,
234	}