]>
Commit | Line | Data |
---|---|---|
1931a55e | 1 | import base64 |
2ad3873f | 2 | import random |
e3ce2b38 | 3 | import re |
1f8b4ab7 | 4 | import urllib.parse |
1931a55e THD |
5 | |
6 | from .common import InfoExtractor | |
7 | from ..utils import ( | |
8 | ExtractorError, | |
1931a55e | 9 | clean_html, |
1f8b4ab7 L |
10 | time_seconds, |
11 | try_call, | |
1931a55e | 12 | unified_timestamp, |
1f8b4ab7 | 13 | update_url_query, |
1931a55e | 14 | ) |
e3ce2b38 | 15 | from ..utils.traversal import traverse_obj |
1931a55e THD |
16 | |
17 | ||
18 | class RadikoBaseIE(InfoExtractor): | |
2ad3873f | 19 | _GEO_BYPASS = False |
1931a55e | 20 | _FULL_KEY = None |
203a06f8 M |
21 | _HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED = ( |
22 | 'https://c-rpaa.smartstream.ne.jp', | |
23 | 'https://si-c-radiko.smartstream.ne.jp', | |
24 | 'https://tf-f-rpaa-radiko.smartstream.ne.jp', | |
25 | 'https://tf-c-rpaa-radiko.smartstream.ne.jp', | |
26 | 'https://si-f-radiko.smartstream.ne.jp', | |
27 | 'https://rpaa.smartstream.ne.jp', | |
28 | ) | |
29 | _HOSTS_FOR_TIME_FREE_FFMPEG_SUPPORTED = ( | |
30 | 'https://rd-wowza-radiko.radiko-cf.com', | |
31 | 'https://radiko.jp', | |
32 | 'https://f-radiko.smartstream.ne.jp', | |
33 | ) | |
34 | # Following URL forcibly connects not Time Free but Live | |
35 | _HOSTS_FOR_LIVE = ( | |
36 | 'https://c-radiko.smartstream.ne.jp', | |
37 | ) | |
1931a55e | 38 | |
2ad3873f | 39 | def _negotiate_token(self): |
1931a55e THD |
40 | _, auth1_handle = self._download_webpage_handle( |
41 | 'https://radiko.jp/v2/api/auth1', None, 'Downloading authentication page', | |
42 | headers={ | |
43 | 'x-radiko-app': 'pc_html5', | |
44 | 'x-radiko-app-version': '0.0.1', | |
45 | 'x-radiko-device': 'pc', | |
46 | 'x-radiko-user': 'dummy_user', | |
47 | }) | |
3d2623a8 | 48 | auth1_header = auth1_handle.headers |
1931a55e THD |
49 | |
50 | auth_token = auth1_header['X-Radiko-AuthToken'] | |
51 | kl = int(auth1_header['X-Radiko-KeyLength']) | |
52 | ko = int(auth1_header['X-Radiko-KeyOffset']) | |
53 | raw_partial_key = self._extract_full_key()[ko:ko + kl] | |
54 | partial_key = base64.b64encode(raw_partial_key).decode() | |
55 | ||
56 | area_id = self._download_webpage( | |
57 | 'https://radiko.jp/v2/api/auth2', None, 'Authenticating', | |
58 | headers={ | |
59 | 'x-radiko-device': 'pc', | |
60 | 'x-radiko-user': 'dummy_user', | |
61 | 'x-radiko-authtoken': auth_token, | |
62 | 'x-radiko-partialkey': partial_key, | |
63 | }).split(',')[0] | |
64 | ||
2ad3873f | 65 | if area_id == 'OUT': |
66 | self.raise_geo_restricted(countries=['JP']) | |
67 | ||
1931a55e | 68 | auth_data = (auth_token, area_id) |
9809740b | 69 | self.cache.store('radiko', 'auth_data', auth_data) |
1931a55e THD |
70 | return auth_data |
71 | ||
2ad3873f | 72 | def _auth_client(self): |
73 | cachedata = self.cache.load('radiko', 'auth_data') | |
74 | if cachedata is not None: | |
75 | response = self._download_webpage( | |
76 | 'https://radiko.jp/v2/api/auth_check', None, 'Checking cached token', expected_status=401, | |
77 | headers={'X-Radiko-AuthToken': cachedata[0], 'X-Radiko-AreaId': cachedata[1]}) | |
78 | if response == 'OK': | |
79 | return cachedata | |
80 | return self._negotiate_token() | |
81 | ||
1931a55e THD |
82 | def _extract_full_key(self): |
83 | if self._FULL_KEY: | |
84 | return self._FULL_KEY | |
85 | ||
86 | jscode = self._download_webpage( | |
87 | 'https://radiko.jp/apps/js/playerCommon.js', None, | |
88 | note='Downloading player js code') | |
89 | full_key = self._search_regex( | |
90 | (r"RadikoJSPlayer\([^,]*,\s*(['\"])pc_html5\1,\s*(['\"])(?P<fullkey>[0-9a-f]+)\2,\s*{"), | |
91 | jscode, 'full key', fatal=False, group='fullkey') | |
92 | ||
93 | if full_key: | |
94 | full_key = full_key.encode() | |
2ad3873f | 95 | else: # use only full key ever known |
1931a55e THD |
96 | full_key = b'bcd151073c03b352e1ef2fd66c32209da9ca0afa' |
97 | ||
98 | self._FULL_KEY = full_key | |
99 | return full_key | |
100 | ||
101 | def _find_program(self, video_id, station, cursor): | |
102 | station_program = self._download_xml( | |
103 | 'https://radiko.jp/v3/program/station/weekly/%s.xml' % station, video_id, | |
104 | note='Downloading radio program for %s station' % station) | |
105 | ||
106 | prog = None | |
107 | for p in station_program.findall('.//prog'): | |
108 | ft_str, to_str = p.attrib['ft'], p.attrib['to'] | |
109 | ft = unified_timestamp(ft_str, False) | |
110 | to = unified_timestamp(to_str, False) | |
111 | if ft <= cursor and cursor < to: | |
112 | prog = p | |
113 | break | |
114 | if not prog: | |
115 | raise ExtractorError('Cannot identify radio program to download!') | |
116 | assert ft, to | |
117 | return prog, station_program, ft, ft_str, to_str | |
118 | ||
119 | def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token, area_id, query): | |
120 | m3u8_playlist_data = self._download_xml( | |
1f8b4ab7 L |
121 | f'https://radiko.jp/v3/station/stream/pc_html5/{station}.xml', video_id, |
122 | note='Downloading stream information') | |
1931a55e THD |
123 | |
124 | formats = [] | |
125 | found = set() | |
2ad3873f | 126 | |
127 | timefree_int = 0 if is_onair else 1 | |
128 | ||
129 | for element in m3u8_playlist_data.findall(f'.//url[@timefree="{timefree_int}"]/playlist_create_url'): | |
130 | pcu = element.text | |
131 | if pcu in found: | |
132 | continue | |
133 | found.add(pcu) | |
203a06f8 | 134 | playlist_url = update_url_query(pcu, { |
1931a55e THD |
135 | 'station_id': station, |
136 | **query, | |
137 | 'l': '15', | |
2ad3873f | 138 | 'lsid': ''.join(random.choices('0123456789abcdef', k=32)), |
1931a55e THD |
139 | 'type': 'b', |
140 | }) | |
1931a55e THD |
141 | |
142 | time_to_skip = None if is_onair else cursor - ft | |
143 | ||
1f8b4ab7 | 144 | domain = urllib.parse.urlparse(playlist_url).netloc |
1931a55e THD |
145 | subformats = self._extract_m3u8_formats( |
146 | playlist_url, video_id, ext='m4a', | |
1f8b4ab7 L |
147 | live=True, fatal=False, m3u8_id=domain, |
148 | note=f'Downloading m3u8 information from {domain}', | |
1931a55e THD |
149 | headers={ |
150 | 'X-Radiko-AreaId': area_id, | |
151 | 'X-Radiko-AuthToken': auth_token, | |
152 | }) | |
153 | for sf in subformats: | |
a5387729 | 154 | if (is_onair ^ pcu.startswith(self._HOSTS_FOR_LIVE)) or ( |
155 | not is_onair and pcu.startswith(self._HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED)): | |
203a06f8 M |
156 | sf['preference'] = -100 |
157 | sf['format_note'] = 'not preferred' | |
2ad3873f | 158 | if not is_onair and timefree_int == 1 and time_to_skip: |
b9316642 | 159 | sf['downloader_options'] = {'ffmpeg_args': ['-ss', str(time_to_skip)]} |
1931a55e THD |
160 | formats.extend(subformats) |
161 | ||
1931a55e THD |
162 | return formats |
163 | ||
e3ce2b38 Y |
164 | def _extract_performers(self, prog): |
165 | performers = traverse_obj(prog, ( | |
166 | 'pfm/text()', ..., {lambda x: re.split(r'[//、 ,,]', x)}, ..., {str.strip})) | |
167 | # TODO: change 'artist' fields to 'artists' and return traversal list instead of str | |
168 | return ', '.join(performers) or None | |
169 | ||
1931a55e THD |
170 | |
171 | class RadikoIE(RadikoBaseIE): | |
172 | _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<id>\d+)' | |
173 | ||
174 | _TESTS = [{ | |
175 | # QRR (文化放送) station provides <desc> | |
176 | 'url': 'https://radiko.jp/#!/ts/QRR/20210425101300', | |
177 | 'only_matching': True, | |
178 | }, { | |
179 | # FMT (TOKYO FM) station does not provide <desc> | |
180 | 'url': 'https://radiko.jp/#!/ts/FMT/20210810150000', | |
181 | 'only_matching': True, | |
182 | }, { | |
183 | 'url': 'https://radiko.jp/#!/ts/JOAK-FM/20210509090000', | |
184 | 'only_matching': True, | |
185 | }] | |
186 | ||
187 | def _real_extract(self, url): | |
188 | station, video_id = self._match_valid_url(url).groups() | |
189 | vid_int = unified_timestamp(video_id, False) | |
1931a55e THD |
190 | prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int) |
191 | ||
2ad3873f | 192 | auth_token, area_id = self._auth_client() |
1931a55e THD |
193 | |
194 | return { | |
195 | 'id': video_id, | |
1f8b4ab7 | 196 | 'title': try_call(lambda: prog.find('title').text), |
e3ce2b38 | 197 | 'artist': self._extract_performers(prog), |
1f8b4ab7 L |
198 | 'description': clean_html(try_call(lambda: prog.find('info').text)), |
199 | 'uploader': try_call(lambda: station_program.find('.//name').text), | |
1931a55e THD |
200 | 'uploader_id': station, |
201 | 'timestamp': vid_int, | |
e3ce2b38 | 202 | 'duration': try_call(lambda: unified_timestamp(radio_end, False) - unified_timestamp(radio_begin, False)), |
1931a55e | 203 | 'is_live': True, |
2ad3873f | 204 | 'formats': self._extract_formats( |
205 | video_id=video_id, station=station, is_onair=False, | |
206 | ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id, | |
207 | query={ | |
208 | 'start_at': radio_begin, | |
209 | 'ft': radio_begin, | |
210 | 'end_at': radio_end, | |
211 | 'to': radio_end, | |
212 | 'seek': video_id | |
213 | } | |
214 | ), | |
1931a55e THD |
215 | } |
216 | ||
217 | ||
218 | class RadikoRadioIE(RadikoBaseIE): | |
219 | _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/live/(?P<id>[A-Z0-9-]+)' | |
220 | ||
221 | _TESTS = [{ | |
222 | # QRR (文化放送) station provides <desc> | |
223 | 'url': 'https://radiko.jp/#!/live/QRR', | |
224 | 'only_matching': True, | |
225 | }, { | |
226 | # FMT (TOKYO FM) station does not provide <desc> | |
227 | 'url': 'https://radiko.jp/#!/live/FMT', | |
228 | 'only_matching': True, | |
229 | }, { | |
230 | 'url': 'https://radiko.jp/#!/live/JOAK-FM', | |
231 | 'only_matching': True, | |
232 | }] | |
233 | ||
234 | def _real_extract(self, url): | |
235 | station = self._match_id(url) | |
236 | self.report_warning('Downloader will not stop at the end of the program! Press Ctrl+C to stop') | |
237 | ||
238 | auth_token, area_id = self._auth_client() | |
239 | # get current time in JST (GMT+9:00 w/o DST) | |
1f8b4ab7 | 240 | vid_now = time_seconds(hours=9) |
1931a55e THD |
241 | |
242 | prog, station_program, ft, _, _ = self._find_program(station, station, vid_now) | |
243 | ||
244 | title = prog.find('title').text | |
245 | description = clean_html(prog.find('info').text) | |
246 | station_name = station_program.find('.//name').text | |
247 | ||
248 | formats = self._extract_formats( | |
249 | video_id=station, station=station, is_onair=True, | |
250 | ft=ft, cursor=vid_now, auth_token=auth_token, area_id=area_id, | |
251 | query={}) | |
252 | ||
253 | return { | |
254 | 'id': station, | |
255 | 'title': title, | |
e3ce2b38 | 256 | 'artist': self._extract_performers(prog), |
1931a55e THD |
257 | 'description': description, |
258 | 'uploader': station_name, | |
259 | 'uploader_id': station, | |
260 | 'timestamp': ft, | |
261 | 'formats': formats, | |
262 | 'is_live': True, | |
263 | } |