yt_dlp/extractor/radiko.py

   1 import base64
   2 import re
   3 import urllib.parse
   4
   5 from .common import InfoExtractor
   6 from ..utils import (
   7     ExtractorError,
   8     clean_html,
   9     time_seconds,
  10     try_call,
  11     unified_timestamp,
  12     update_url_query,
  13 )
  14
  15
  16 class RadikoBaseIE(InfoExtractor):
  17     _FULL_KEY = None
  18
  19     def _auth_client(self):
  20         _, auth1_handle = self._download_webpage_handle(
  21             'https://radiko.jp/v2/api/auth1', None, 'Downloading authentication page',
  22             headers={
  23                 'x-radiko-app': 'pc_html5',
  24                 'x-radiko-app-version': '0.0.1',
  25                 'x-radiko-device': 'pc',
  26                 'x-radiko-user': 'dummy_user',
  27             })
  28         auth1_header = auth1_handle.info()
  29
  30         auth_token = auth1_header['X-Radiko-AuthToken']
  31         kl = int(auth1_header['X-Radiko-KeyLength'])
  32         ko = int(auth1_header['X-Radiko-KeyOffset'])
  33         raw_partial_key = self._extract_full_key()[ko:ko + kl]
  34         partial_key = base64.b64encode(raw_partial_key).decode()
  35
  36         area_id = self._download_webpage(
  37             'https://radiko.jp/v2/api/auth2', None, 'Authenticating',
  38             headers={
  39                 'x-radiko-device': 'pc',
  40                 'x-radiko-user': 'dummy_user',
  41                 'x-radiko-authtoken': auth_token,
  42                 'x-radiko-partialkey': partial_key,
  43             }).split(',')[0]
  44
  45         auth_data = (auth_token, area_id)
  46         self.cache.store('radiko', 'auth_data', auth_data)
  47         return auth_data
  48
  49     def _extract_full_key(self):
  50         if self._FULL_KEY:
  51             return self._FULL_KEY
  52
  53         jscode = self._download_webpage(
  54             'https://radiko.jp/apps/js/playerCommon.js', None,
  55             note='Downloading player js code')
  56         full_key = self._search_regex(
  57             (r"RadikoJSPlayer\([^,]*,\s*(['\"])pc_html5\1,\s*(['\"])(?P<fullkey>[0-9a-f]+)\2,\s*{"),
  58             jscode, 'full key', fatal=False, group='fullkey')
  59
  60         if full_key:
  61             full_key = full_key.encode()
  62         else:  # use full key ever known
  63             full_key = b'bcd151073c03b352e1ef2fd66c32209da9ca0afa'
  64
  65         self._FULL_KEY = full_key
  66         return full_key
  67
  68     def _find_program(self, video_id, station, cursor):
  69         station_program = self._download_xml(
  70             'https://radiko.jp/v3/program/station/weekly/%s.xml' % station, video_id,
  71             note='Downloading radio program for %s station' % station)
  72
  73         prog = None
  74         for p in station_program.findall('.//prog'):
  75             ft_str, to_str = p.attrib['ft'], p.attrib['to']
  76             ft = unified_timestamp(ft_str, False)
  77             to = unified_timestamp(to_str, False)
  78             if ft <= cursor and cursor < to:
  79                 prog = p
  80                 break
  81         if not prog:
  82             raise ExtractorError('Cannot identify radio program to download!')
  83         assert ft, to
  84         return prog, station_program, ft, ft_str, to_str
  85
  86     def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token, area_id, query):
  87         m3u8_playlist_data = self._download_xml(
  88             f'https://radiko.jp/v3/station/stream/pc_html5/{station}.xml', video_id,
  89             note='Downloading stream information')
  90         m3u8_urls = m3u8_playlist_data.findall('.//url')
  91
  92         formats = []
  93         found = set()
  94         for url_tag in m3u8_urls:
  95             pcu = url_tag.find('playlist_create_url')
  96             url_attrib = url_tag.attrib
  97             playlist_url = update_url_query(pcu.text, {
  98                 'station_id': station,
  99                 **query,
 100                 'l': '15',
 101                 'lsid': '88ecea37e968c1f17d5413312d9f8003',
 102                 'type': 'b',
 103             })
 104             if playlist_url in found:
 105                 continue
 106             else:
 107                 found.add(playlist_url)
 108
 109             time_to_skip = None if is_onair else cursor - ft
 110
 111             domain = urllib.parse.urlparse(playlist_url).netloc
 112             subformats = self._extract_m3u8_formats(
 113                 playlist_url, video_id, ext='m4a',
 114                 live=True, fatal=False, m3u8_id=domain,
 115                 note=f'Downloading m3u8 information from {domain}',
 116                 headers={
 117                     'X-Radiko-AreaId': area_id,
 118                     'X-Radiko-AuthToken': auth_token,
 119                 })
 120             for sf in subformats:
 121                 if re.fullmatch(r'[cf]-radiko\.smartstream\.ne\.jp', domain):
 122                     # Prioritize live radio vs playback based on extractor
 123                     sf['preference'] = 100 if is_onair else -100
 124                 if not is_onair and url_attrib['timefree'] == '1' and time_to_skip:
 125                     sf['downloader_options'] = {'ffmpeg_args': ['-ss', time_to_skip]}
 126             formats.extend(subformats)
 127
 128         self._sort_formats(formats)
 129         return formats
 130
 131
 132 class RadikoIE(RadikoBaseIE):
 133     _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<id>\d+)'
 134
 135     _TESTS = [{
 136         # QRR (文化放送) station provides <desc>
 137         'url': 'https://radiko.jp/#!/ts/QRR/20210425101300',
 138         'only_matching': True,
 139     }, {
 140         # FMT (TOKYO FM) station does not provide <desc>
 141         'url': 'https://radiko.jp/#!/ts/FMT/20210810150000',
 142         'only_matching': True,
 143     }, {
 144         'url': 'https://radiko.jp/#!/ts/JOAK-FM/20210509090000',
 145         'only_matching': True,
 146     }]
 147
 148     def _real_extract(self, url):
 149         station, video_id = self._match_valid_url(url).groups()
 150         vid_int = unified_timestamp(video_id, False)
 151         prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int)
 152
 153         auth_cache = self.cache.load('radiko', 'auth_data')
 154         for attempt in range(2):
 155             auth_token, area_id = (not attempt and auth_cache) or self._auth_client()
 156             formats = self._extract_formats(
 157                 video_id=video_id, station=station, is_onair=False,
 158                 ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id,
 159                 query={
 160                     'start_at': radio_begin,
 161                     'ft': radio_begin,
 162                     'end_at': radio_end,
 163                     'to': radio_end,
 164                     'seek': video_id,
 165                 })
 166             if formats:
 167                 break
 168
 169         return {
 170             'id': video_id,
 171             'title': try_call(lambda: prog.find('title').text),
 172             'description': clean_html(try_call(lambda: prog.find('info').text)),
 173             'uploader': try_call(lambda: station_program.find('.//name').text),
 174             'uploader_id': station,
 175             'timestamp': vid_int,
 176             'formats': formats,
 177             'is_live': True,
 178         }
 179
 180
 181 class RadikoRadioIE(RadikoBaseIE):
 182     _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/live/(?P<id>[A-Z0-9-]+)'
 183
 184     _TESTS = [{
 185         # QRR (文化放送) station provides <desc>
 186         'url': 'https://radiko.jp/#!/live/QRR',
 187         'only_matching': True,
 188     }, {
 189         # FMT (TOKYO FM) station does not provide <desc>
 190         'url': 'https://radiko.jp/#!/live/FMT',
 191         'only_matching': True,
 192     }, {
 193         'url': 'https://radiko.jp/#!/live/JOAK-FM',
 194         'only_matching': True,
 195     }]
 196
 197     def _real_extract(self, url):
 198         station = self._match_id(url)
 199         self.report_warning('Downloader will not stop at the end of the program! Press Ctrl+C to stop')
 200
 201         auth_token, area_id = self._auth_client()
 202         # get current time in JST (GMT+9:00 w/o DST)
 203         vid_now = time_seconds(hours=9)
 204
 205         prog, station_program, ft, _, _ = self._find_program(station, station, vid_now)
 206
 207         title = prog.find('title').text
 208         description = clean_html(prog.find('info').text)
 209         station_name = station_program.find('.//name').text
 210
 211         formats = self._extract_formats(
 212             video_id=station, station=station, is_onair=True,
 213             ft=ft, cursor=vid_now, auth_token=auth_token, area_id=area_id,
 214             query={})
 215
 216         return {
 217             'id': station,
 218             'title': title,
 219             'description': description,
 220             'uploader': station_name,
 221             'uploader_id': station,
 222             'timestamp': ft,
 223             'formats': formats,
 224             'is_live': True,
 225         }