]>
Commit | Line | Data |
---|---|---|
1 | import base64 | |
2 | import binascii | |
3 | import hashlib | |
4 | import hmac | |
5 | import io | |
6 | import json | |
7 | import re | |
8 | import struct | |
9 | import time | |
10 | import urllib.response | |
11 | import uuid | |
12 | ||
13 | from .common import InfoExtractor | |
14 | from ..aes import aes_ecb_decrypt | |
15 | from ..compat import compat_urllib_parse_urlparse, compat_urllib_request | |
16 | from ..utils import ( | |
17 | ExtractorError, | |
18 | bytes_to_intlist, | |
19 | decode_base, | |
20 | int_or_none, | |
21 | intlist_to_bytes, | |
22 | request_to_url, | |
23 | time_seconds, | |
24 | traverse_obj, | |
25 | update_url_query, | |
26 | urljoin, | |
27 | ) | |
28 | ||
29 | # NOTE: network handler related code is temporary thing until network stack overhaul PRs are merged (#2861/#2862) | |
30 | ||
31 | ||
32 | def add_opener(ydl, handler): | |
33 | ''' Add a handler for opening URLs, like _download_webpage ''' | |
34 | # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426 | |
35 | # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605 | |
36 | assert isinstance(ydl._opener, compat_urllib_request.OpenerDirector) | |
37 | ydl._opener.add_handler(handler) | |
38 | ||
39 | ||
40 | def remove_opener(ydl, handler): | |
41 | ''' | |
42 | Remove handler(s) for opening URLs | |
43 | @param handler Either handler object itself or handler type. | |
44 | Specifying handler type will remove all handler which isinstance returns True. | |
45 | ''' | |
46 | # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426 | |
47 | # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605 | |
48 | opener = ydl._opener | |
49 | assert isinstance(ydl._opener, compat_urllib_request.OpenerDirector) | |
50 | if isinstance(handler, (type, tuple)): | |
51 | find_cp = lambda x: isinstance(x, handler) | |
52 | else: | |
53 | find_cp = lambda x: x is handler | |
54 | ||
55 | removed = [] | |
56 | for meth in dir(handler): | |
57 | if meth in ["redirect_request", "do_open", "proxy_open"]: | |
58 | # oops, coincidental match | |
59 | continue | |
60 | ||
61 | i = meth.find("_") | |
62 | protocol = meth[:i] | |
63 | condition = meth[i + 1:] | |
64 | ||
65 | if condition.startswith("error"): | |
66 | j = condition.find("_") + i + 1 | |
67 | kind = meth[j + 1:] | |
68 | try: | |
69 | kind = int(kind) | |
70 | except ValueError: | |
71 | pass | |
72 | lookup = opener.handle_error.get(protocol, {}) | |
73 | opener.handle_error[protocol] = lookup | |
74 | elif condition == "open": | |
75 | kind = protocol | |
76 | lookup = opener.handle_open | |
77 | elif condition == "response": | |
78 | kind = protocol | |
79 | lookup = opener.process_response | |
80 | elif condition == "request": | |
81 | kind = protocol | |
82 | lookup = opener.process_request | |
83 | else: | |
84 | continue | |
85 | ||
86 | handlers = lookup.setdefault(kind, []) | |
87 | if handlers: | |
88 | handlers[:] = [x for x in handlers if not find_cp(x)] | |
89 | ||
90 | removed.append(x for x in handlers if find_cp(x)) | |
91 | ||
92 | if removed: | |
93 | for x in opener.handlers: | |
94 | if find_cp(x): | |
95 | x.add_parent(None) | |
96 | opener.handlers[:] = [x for x in opener.handlers if not find_cp(x)] | |
97 | ||
98 | ||
99 | class AbemaLicenseHandler(compat_urllib_request.BaseHandler): | |
100 | handler_order = 499 | |
101 | STRTABLE = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' | |
102 | HKEY = b'3AF0298C219469522A313570E8583005A642E73EDD58E3EA2FB7339D3DF1597E' | |
103 | ||
104 | def __init__(self, ie: 'AbemaTVIE'): | |
105 | # the protcol that this should really handle is 'abematv-license://' | |
106 | # abematv_license_open is just a placeholder for development purposes | |
107 | # ref. https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Lib/urllib/request.py#L510 | |
108 | setattr(self, 'abematv-license_open', getattr(self, 'abematv_license_open')) | |
109 | self.ie = ie | |
110 | ||
111 | def _get_videokey_from_ticket(self, ticket): | |
112 | to_show = self.ie._downloader.params.get('verbose', False) | |
113 | media_token = self.ie._get_media_token(to_show=to_show) | |
114 | ||
115 | license_response = self.ie._download_json( | |
116 | 'https://license.abema.io/abematv-hls', None, note='Requesting playback license' if to_show else False, | |
117 | query={'t': media_token}, | |
118 | data=json.dumps({ | |
119 | 'kv': 'a', | |
120 | 'lt': ticket | |
121 | }).encode('utf-8'), | |
122 | headers={ | |
123 | 'Content-Type': 'application/json', | |
124 | }) | |
125 | ||
126 | res = decode_base(license_response['k'], self.STRTABLE) | |
127 | encvideokey = bytes_to_intlist(struct.pack('>QQ', res >> 64, res & 0xffffffffffffffff)) | |
128 | ||
129 | h = hmac.new( | |
130 | binascii.unhexlify(self.HKEY), | |
131 | (license_response['cid'] + self.ie._DEVICE_ID).encode('utf-8'), | |
132 | digestmod=hashlib.sha256) | |
133 | enckey = bytes_to_intlist(h.digest()) | |
134 | ||
135 | return intlist_to_bytes(aes_ecb_decrypt(encvideokey, enckey)) | |
136 | ||
137 | def abematv_license_open(self, url): | |
138 | url = request_to_url(url) | |
139 | ticket = compat_urllib_parse_urlparse(url).netloc | |
140 | response_data = self._get_videokey_from_ticket(ticket) | |
141 | return urllib.response.addinfourl(io.BytesIO(response_data), headers={ | |
142 | 'Content-Length': len(response_data), | |
143 | }, url=url, code=200) | |
144 | ||
145 | ||
146 | class AbemaTVBaseIE(InfoExtractor): | |
147 | def _extract_breadcrumb_list(self, webpage, video_id): | |
148 | for jld in re.finditer( | |
149 | r'(?is)</span></li></ul><script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>', | |
150 | webpage): | |
151 | jsonld = self._parse_json(jld.group('json_ld'), video_id, fatal=False) | |
152 | if jsonld: | |
153 | if jsonld.get('@type') != 'BreadcrumbList': | |
154 | continue | |
155 | trav = traverse_obj(jsonld, ('itemListElement', ..., 'name')) | |
156 | if trav: | |
157 | return trav | |
158 | return [] | |
159 | ||
160 | ||
161 | class AbemaTVIE(AbemaTVBaseIE): | |
162 | _VALID_URL = r'https?://abema\.tv/(?P<type>now-on-air|video/episode|channels/.+?/slots)/(?P<id>[^?/]+)' | |
163 | _NETRC_MACHINE = 'abematv' | |
164 | _TESTS = [{ | |
165 | 'url': 'https://abema.tv/video/episode/194-25_s2_p1', | |
166 | 'info_dict': { | |
167 | 'id': '194-25_s2_p1', | |
168 | 'title': '第1話 「チーズケーキ」 「モーニング再び」', | |
169 | 'series': '異世界食堂2', | |
170 | 'series_number': 2, | |
171 | 'episode': '第1話 「チーズケーキ」 「モーニング再び」', | |
172 | 'episode_number': 1, | |
173 | }, | |
174 | 'skip': 'expired', | |
175 | }, { | |
176 | 'url': 'https://abema.tv/channels/anime-live2/slots/E8tvAnMJ7a9a5d', | |
177 | 'info_dict': { | |
178 | 'id': 'E8tvAnMJ7a9a5d', | |
179 | 'title': 'ゆるキャン△ SEASON2 全話一挙【無料ビデオ72時間】', | |
180 | 'series': 'ゆるキャン△ SEASON2', | |
181 | 'episode': 'ゆるキャン△ SEASON2 全話一挙【無料ビデオ72時間】', | |
182 | 'series_number': 2, | |
183 | 'episode_number': 1, | |
184 | 'description': 'md5:9c5a3172ae763278f9303922f0ea5b17', | |
185 | }, | |
186 | 'skip': 'expired', | |
187 | }, { | |
188 | 'url': 'https://abema.tv/video/episode/87-877_s1282_p31047', | |
189 | 'info_dict': { | |
190 | 'id': 'E8tvAnMJ7a9a5d', | |
191 | 'title': '第5話『光射す』', | |
192 | 'description': 'md5:56d4fc1b4f7769ded5f923c55bb4695d', | |
193 | 'thumbnail': r're:https://hayabusa\.io/.+', | |
194 | 'series': '相棒', | |
195 | 'episode': '第5話『光射す』', | |
196 | }, | |
197 | 'skip': 'expired', | |
198 | }, { | |
199 | 'url': 'https://abema.tv/now-on-air/abema-anime', | |
200 | 'info_dict': { | |
201 | 'id': 'abema-anime', | |
202 | # this varies | |
203 | # 'title': '女子高生の無駄づかい 全話一挙【無料ビデオ72時間】', | |
204 | 'description': 'md5:55f2e61f46a17e9230802d7bcc913d5f', | |
205 | 'is_live': True, | |
206 | }, | |
207 | 'skip': 'Not supported until yt-dlp implements native live downloader OR AbemaTV can start a local HTTP server', | |
208 | }] | |
209 | _USERTOKEN = None | |
210 | _DEVICE_ID = None | |
211 | _TIMETABLE = None | |
212 | _MEDIATOKEN = None | |
213 | ||
214 | _SECRETKEY = b'v+Gjs=25Aw5erR!J8ZuvRrCx*rGswhB&qdHd_SYerEWdU&a?3DzN9BRbp5KwY4hEmcj5#fykMjJ=AuWz5GSMY-d@H7DMEh3M@9n2G552Us$$k9cD=3TxwWe86!x#Zyhe' | |
215 | ||
216 | def _generate_aks(self, deviceid): | |
217 | deviceid = deviceid.encode('utf-8') | |
218 | # add 1 hour and then drop minute and secs | |
219 | ts_1hour = int((time_seconds(hours=9) // 3600 + 1) * 3600) | |
220 | time_struct = time.gmtime(ts_1hour) | |
221 | ts_1hour_str = str(ts_1hour).encode('utf-8') | |
222 | ||
223 | tmp = None | |
224 | ||
225 | def mix_once(nonce): | |
226 | nonlocal tmp | |
227 | h = hmac.new(self._SECRETKEY, digestmod=hashlib.sha256) | |
228 | h.update(nonce) | |
229 | tmp = h.digest() | |
230 | ||
231 | def mix_tmp(count): | |
232 | nonlocal tmp | |
233 | for i in range(count): | |
234 | mix_once(tmp) | |
235 | ||
236 | def mix_twist(nonce): | |
237 | nonlocal tmp | |
238 | mix_once(base64.urlsafe_b64encode(tmp).rstrip(b'=') + nonce) | |
239 | ||
240 | mix_once(self._SECRETKEY) | |
241 | mix_tmp(time_struct.tm_mon) | |
242 | mix_twist(deviceid) | |
243 | mix_tmp(time_struct.tm_mday % 5) | |
244 | mix_twist(ts_1hour_str) | |
245 | mix_tmp(time_struct.tm_hour % 5) | |
246 | ||
247 | return base64.urlsafe_b64encode(tmp).rstrip(b'=').decode('utf-8') | |
248 | ||
249 | def _get_device_token(self): | |
250 | if self._USERTOKEN: | |
251 | return self._USERTOKEN | |
252 | ||
253 | self._DEVICE_ID = str(uuid.uuid4()) | |
254 | aks = self._generate_aks(self._DEVICE_ID) | |
255 | user_data = self._download_json( | |
256 | 'https://api.abema.io/v1/users', None, note='Authorizing', | |
257 | data=json.dumps({ | |
258 | 'deviceId': self._DEVICE_ID, | |
259 | 'applicationKeySecret': aks, | |
260 | }).encode('utf-8'), | |
261 | headers={ | |
262 | 'Content-Type': 'application/json', | |
263 | }) | |
264 | self._USERTOKEN = user_data['token'] | |
265 | ||
266 | # don't allow adding it 2 times or more, though it's guarded | |
267 | remove_opener(self._downloader, AbemaLicenseHandler) | |
268 | add_opener(self._downloader, AbemaLicenseHandler(self)) | |
269 | ||
270 | return self._USERTOKEN | |
271 | ||
272 | def _get_media_token(self, invalidate=False, to_show=True): | |
273 | if not invalidate and self._MEDIATOKEN: | |
274 | return self._MEDIATOKEN | |
275 | ||
276 | self._MEDIATOKEN = self._download_json( | |
277 | 'https://api.abema.io/v1/media/token', None, note='Fetching media token' if to_show else False, | |
278 | query={ | |
279 | 'osName': 'android', | |
280 | 'osVersion': '6.0.1', | |
281 | 'osLang': 'ja_JP', | |
282 | 'osTimezone': 'Asia/Tokyo', | |
283 | 'appId': 'tv.abema', | |
284 | 'appVersion': '3.27.1' | |
285 | }, headers={ | |
286 | 'Authorization': 'bearer ' + self._get_device_token() | |
287 | })['token'] | |
288 | ||
289 | return self._MEDIATOKEN | |
290 | ||
291 | def _perform_login(self, username, password): | |
292 | if '@' in username: # don't strictly check if it's email address or not | |
293 | ep, method = 'user/email', 'email' | |
294 | else: | |
295 | ep, method = 'oneTimePassword', 'userId' | |
296 | ||
297 | login_response = self._download_json( | |
298 | f'https://api.abema.io/v1/auth/{ep}', None, note='Logging in', | |
299 | data=json.dumps({ | |
300 | method: username, | |
301 | 'password': password | |
302 | }).encode('utf-8'), headers={ | |
303 | 'Authorization': 'bearer ' + self._get_device_token(), | |
304 | 'Origin': 'https://abema.tv', | |
305 | 'Referer': 'https://abema.tv/', | |
306 | 'Content-Type': 'application/json', | |
307 | }) | |
308 | ||
309 | self._USERTOKEN = login_response['token'] | |
310 | self._get_media_token(True) | |
311 | ||
312 | def _real_extract(self, url): | |
313 | # starting download using infojson from this extractor is undefined behavior, | |
314 | # and never be fixed in the future; you must trigger downloads by directly specifing URL. | |
315 | # (unless there's a way to hook before downloading by extractor) | |
316 | video_id, video_type = self._match_valid_url(url).group('id', 'type') | |
317 | headers = { | |
318 | 'Authorization': 'Bearer ' + self._get_device_token(), | |
319 | } | |
320 | video_type = video_type.split('/')[-1] | |
321 | ||
322 | webpage = self._download_webpage(url, video_id) | |
323 | canonical_url = self._search_regex( | |
324 | r'<link\s+rel="canonical"\s*href="(.+?)"', webpage, 'canonical URL', | |
325 | default=url) | |
326 | info = self._search_json_ld(webpage, video_id, default={}) | |
327 | ||
328 | title = self._search_regex( | |
329 | r'<span\s*class=".+?EpisodeTitleBlock__title">(.+?)</span>', webpage, 'title', default=None) | |
330 | if not title: | |
331 | jsonld = None | |
332 | for jld in re.finditer( | |
333 | r'(?is)<span\s*class="com-m-Thumbnail__image">(?:</span>)?<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>', | |
334 | webpage): | |
335 | jsonld = self._parse_json(jld.group('json_ld'), video_id, fatal=False) | |
336 | if jsonld: | |
337 | break | |
338 | if jsonld: | |
339 | title = jsonld.get('caption') | |
340 | if not title and video_type == 'now-on-air': | |
341 | if not self._TIMETABLE: | |
342 | # cache the timetable because it goes to 5MiB in size (!!) | |
343 | self._TIMETABLE = self._download_json( | |
344 | 'https://api.abema.io/v1/timetable/dataSet?debug=false', video_id, | |
345 | headers=headers) | |
346 | now = time_seconds(hours=9) | |
347 | for slot in self._TIMETABLE.get('slots', []): | |
348 | if slot.get('channelId') != video_id: | |
349 | continue | |
350 | if slot['startAt'] <= now and now < slot['endAt']: | |
351 | title = slot['title'] | |
352 | break | |
353 | ||
354 | # read breadcrumb on top of page | |
355 | breadcrumb = self._extract_breadcrumb_list(webpage, video_id) | |
356 | if breadcrumb: | |
357 | # breadcrumb list translates to: (example is 1st test for this IE) | |
358 | # Home > Anime (genre) > Isekai Shokudo 2 (series name) > Episode 1 "Cheese cakes" "Morning again" (episode title) | |
359 | # hence this works | |
360 | info['series'] = breadcrumb[-2] | |
361 | info['episode'] = breadcrumb[-1] | |
362 | if not title: | |
363 | title = info['episode'] | |
364 | ||
365 | description = self._html_search_regex( | |
366 | (r'<p\s+class="com-video-EpisodeDetailsBlock__content"><span\s+class=".+?">(.+?)</span></p><div', | |
367 | r'<span\s+class=".+?SlotSummary.+?">(.+?)</span></div><div',), | |
368 | webpage, 'description', default=None, group=1) | |
369 | if not description: | |
370 | og_desc = self._html_search_meta( | |
371 | ('description', 'og:description', 'twitter:description'), webpage) | |
372 | if og_desc: | |
373 | description = re.sub(r'''(?sx) | |
374 | ^(.+?)(?: | |
375 | アニメの動画を無料で見るならABEMA!| # anime | |
376 | 等、.+ # applies for most of categories | |
377 | )? | |
378 | ''', r'\1', og_desc) | |
379 | ||
380 | # canonical URL may contain series and episode number | |
381 | mobj = re.search(r's(\d+)_p(\d+)$', canonical_url) | |
382 | if mobj: | |
383 | seri = int_or_none(mobj.group(1), default=float('inf')) | |
384 | epis = int_or_none(mobj.group(2), default=float('inf')) | |
385 | info['series_number'] = seri if seri < 100 else None | |
386 | # some anime like Detective Conan (though not available in AbemaTV) | |
387 | # has more than 1000 episodes (1026 as of 2021/11/15) | |
388 | info['episode_number'] = epis if epis < 2000 else None | |
389 | ||
390 | is_live, m3u8_url = False, None | |
391 | if video_type == 'now-on-air': | |
392 | is_live = True | |
393 | channel_url = 'https://api.abema.io/v1/channels' | |
394 | if video_id == 'news-global': | |
395 | channel_url = update_url_query(channel_url, {'division': '1'}) | |
396 | onair_channels = self._download_json(channel_url, video_id) | |
397 | for ch in onair_channels['channels']: | |
398 | if video_id == ch['id']: | |
399 | m3u8_url = ch['playback']['hls'] | |
400 | break | |
401 | else: | |
402 | raise ExtractorError(f'Cannot find on-air {video_id} channel.', expected=True) | |
403 | elif video_type == 'episode': | |
404 | api_response = self._download_json( | |
405 | f'https://api.abema.io/v1/video/programs/{video_id}', video_id, | |
406 | note='Checking playability', | |
407 | headers=headers) | |
408 | ondemand_types = traverse_obj(api_response, ('terms', ..., 'onDemandType'), default=[]) | |
409 | if 3 not in ondemand_types: | |
410 | # cannot acquire decryption key for these streams | |
411 | self.report_warning('This is a premium-only stream') | |
412 | ||
413 | m3u8_url = f'https://vod-abematv.akamaized.net/program/{video_id}/playlist.m3u8' | |
414 | elif video_type == 'slots': | |
415 | api_response = self._download_json( | |
416 | f'https://api.abema.io/v1/media/slots/{video_id}', video_id, | |
417 | note='Checking playability', | |
418 | headers=headers) | |
419 | if not traverse_obj(api_response, ('slot', 'flags', 'timeshiftFree'), default=False): | |
420 | self.report_warning('This is a premium-only stream') | |
421 | ||
422 | m3u8_url = f'https://vod-abematv.akamaized.net/slot/{video_id}/playlist.m3u8' | |
423 | else: | |
424 | raise ExtractorError('Unreachable') | |
425 | ||
426 | if is_live: | |
427 | self.report_warning("This is a livestream; yt-dlp doesn't support downloading natively, but FFmpeg cannot handle m3u8 manifests from AbemaTV") | |
428 | self.report_warning('Please consider using Streamlink to download these streams (https://github.com/streamlink/streamlink)') | |
429 | formats = self._extract_m3u8_formats( | |
430 | m3u8_url, video_id, ext='mp4', live=is_live) | |
431 | ||
432 | info.update({ | |
433 | 'id': video_id, | |
434 | 'title': title, | |
435 | 'description': description, | |
436 | 'formats': formats, | |
437 | 'is_live': is_live, | |
438 | }) | |
439 | return info | |
440 | ||
441 | ||
442 | class AbemaTVTitleIE(AbemaTVBaseIE): | |
443 | _VALID_URL = r'https?://abema\.tv/video/title/(?P<id>[^?/]+)' | |
444 | ||
445 | _TESTS = [{ | |
446 | 'url': 'https://abema.tv/video/title/90-1597', | |
447 | 'info_dict': { | |
448 | 'id': '90-1597', | |
449 | 'title': 'シャッフルアイランド', | |
450 | }, | |
451 | 'playlist_mincount': 2, | |
452 | }, { | |
453 | 'url': 'https://abema.tv/video/title/193-132', | |
454 | 'info_dict': { | |
455 | 'id': '193-132', | |
456 | 'title': '真心が届く~僕とスターのオフィス・ラブ!?~', | |
457 | }, | |
458 | 'playlist_mincount': 16, | |
459 | }] | |
460 | ||
461 | def _real_extract(self, url): | |
462 | video_id = self._match_id(url) | |
463 | webpage = self._download_webpage(url, video_id) | |
464 | ||
465 | playlist_title, breadcrumb = None, self._extract_breadcrumb_list(webpage, video_id) | |
466 | if breadcrumb: | |
467 | playlist_title = breadcrumb[-1] | |
468 | ||
469 | playlist = [ | |
470 | self.url_result(urljoin('https://abema.tv/', mobj.group(1))) | |
471 | for mobj in re.finditer(r'<li\s*class=".+?EpisodeList.+?"><a\s*href="(/[^"]+?)"', webpage)] | |
472 | ||
473 | return self.playlist_result(playlist, playlist_title=playlist_title, playlist_id=video_id) |