13 import urllib
.response
15 from ..utils
.networking
import clean_proxies
16 from .common
import InfoExtractor
17 from ..aes
import aes_ecb_decrypt
31 def add_opener(ydl
, handler
): # FIXME: Create proper API in .networking
32 """Add a handler for opening URLs, like _download_webpage"""
33 # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
34 # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
35 rh
= ydl
._request
_director
.handlers
['Urllib']
36 if 'abematv-license' in rh
._SUPPORTED
_URL
_SCHEMES
:
38 headers
= ydl
.params
['http_headers'].copy()
39 proxies
= ydl
.proxies
.copy()
40 clean_proxies(proxies
, headers
)
41 opener
= rh
._get
_instance
(cookiejar
=ydl
.cookiejar
, proxies
=proxies
)
42 assert isinstance(opener
, urllib
.request
.OpenerDirector
)
43 opener
.add_handler(handler
)
44 rh
._SUPPORTED
_URL
_SCHEMES
= (*rh
._SUPPORTED
_URL
_SCHEMES
, 'abematv-license')
47 class AbemaLicenseHandler(urllib
.request
.BaseHandler
):
49 STRTABLE
= '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
50 HKEY
= b
'3AF0298C219469522A313570E8583005A642E73EDD58E3EA2FB7339D3DF1597E'
52 def __init__(self
, ie
: 'AbemaTVIE'):
53 # the protocol that this should really handle is 'abematv-license://'
54 # abematv_license_open is just a placeholder for development purposes
55 # ref. https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Lib/urllib/request.py#L510
56 setattr(self
, 'abematv-license_open', getattr(self
, 'abematv_license_open', None))
59 def _get_videokey_from_ticket(self
, ticket
):
60 to_show
= self
.ie
.get_param('verbose', False)
61 media_token
= self
.ie
._get
_media
_token
(to_show
=to_show
)
63 license_response
= self
.ie
._download
_json
(
64 'https://license.abema.io/abematv-hls', None, note
='Requesting playback license' if to_show
else False,
65 query
={'t': media_token}
,
71 'Content-Type': 'application/json',
74 res
= decode_base_n(license_response
['k'], table
=self
.STRTABLE
)
75 encvideokey
= bytes_to_intlist(struct
.pack('>QQ', res
>> 64, res
& 0xffffffffffffffff))
78 binascii
.unhexlify(self
.HKEY
),
79 (license_response
['cid'] + self
.ie
._DEVICE
_ID
).encode('utf-8'),
80 digestmod
=hashlib
.sha256
)
81 enckey
= bytes_to_intlist(h
.digest())
83 return intlist_to_bytes(aes_ecb_decrypt(encvideokey
, enckey
))
85 def abematv_license_open(self
, url
):
86 url
= url
.get_full_url() if isinstance(url
, urllib
.request
.Request
) else url
87 ticket
= urllib
.parse
.urlparse(url
).netloc
88 response_data
= self
._get
_videokey
_from
_ticket
(ticket
)
89 return urllib
.response
.addinfourl(io
.BytesIO(response_data
), headers
={
90 'Content-Length': str(len(response_data
)),
94 class AbemaTVBaseIE(InfoExtractor
):
95 _NETRC_MACHINE
= 'abematv'
101 _SECRETKEY
= b
'v+Gjs=25Aw5erR!J8ZuvRrCx*rGswhB&qdHd_SYerEWdU&a?3DzN9BRbp5KwY4hEmcj5#fykMjJ=AuWz5GSMY-d@H7DMEh3M@9n2G552Us$$k9cD=3TxwWe86!x#Zyhe'
104 def _generate_aks(cls
, deviceid
):
105 deviceid
= deviceid
.encode('utf-8')
106 # add 1 hour and then drop minute and secs
107 ts_1hour
= int((time_seconds() // 3600 + 1) * 3600)
108 time_struct
= time
.gmtime(ts_1hour
)
109 ts_1hour_str
= str(ts_1hour
).encode('utf-8')
115 h
= hmac
.new(cls
._SECRETKEY
, digestmod
=hashlib
.sha256
)
121 for i
in range(count
):
124 def mix_twist(nonce
):
126 mix_once(base64
.urlsafe_b64encode(tmp
).rstrip(b
'=') + nonce
)
128 mix_once(cls
._SECRETKEY
)
129 mix_tmp(time_struct
.tm_mon
)
131 mix_tmp(time_struct
.tm_mday
% 5)
132 mix_twist(ts_1hour_str
)
133 mix_tmp(time_struct
.tm_hour
% 5)
135 return base64
.urlsafe_b64encode(tmp
).rstrip(b
'=').decode('utf-8')
137 def _get_device_token(self
):
139 return self
._USERTOKEN
141 add_opener(self
._downloader
, AbemaLicenseHandler(self
))
143 username
, _
= self
._get
_login
_info
()
144 auth_cache
= username
and self
.cache
.load(self
._NETRC
_MACHINE
, username
, min_ver
='2024.01.19')
145 AbemaTVBaseIE
._USERTOKEN
= auth_cache
and auth_cache
.get('usertoken')
146 if AbemaTVBaseIE
._USERTOKEN
:
147 # try authentication with locally stored token
149 AbemaTVBaseIE
._DEVICE
_ID
= auth_cache
.get('device_id')
150 self
._get
_media
_token
(True)
152 except ExtractorError
as e
:
153 self
.report_warning(f
'Failed to login with cached user token; obtaining a fresh one ({e})')
155 AbemaTVBaseIE
._DEVICE
_ID
= str(uuid
.uuid4())
156 aks
= self
._generate
_aks
(self
._DEVICE
_ID
)
157 user_data
= self
._download
_json
(
158 'https://api.abema.io/v1/users', None, note
='Authorizing',
160 'deviceId': self
._DEVICE
_ID
,
161 'applicationKeySecret': aks
,
164 'Content-Type': 'application/json',
166 AbemaTVBaseIE
._USERTOKEN
= user_data
['token']
168 return self
._USERTOKEN
170 def _get_media_token(self
, invalidate
=False, to_show
=True):
171 if not invalidate
and self
._MEDIATOKEN
:
172 return self
._MEDIATOKEN
174 AbemaTVBaseIE
._MEDIATOKEN
= self
._download
_json
(
175 'https://api.abema.io/v1/media/token', None, note
='Fetching media token' if to_show
else False,
178 'osVersion': '6.0.1',
180 'osTimezone': 'Asia/Tokyo',
182 'appVersion': '3.27.1'
184 'Authorization': f
'bearer {self._get_device_token()}',
187 return self
._MEDIATOKEN
189 def _perform_login(self
, username
, password
):
190 self
._get
_device
_token
()
191 if self
.cache
.load(self
._NETRC
_MACHINE
, username
, min_ver
='2024.01.19') and self
._get
_media
_token
():
192 self
.write_debug('Skipping logging in')
195 if '@' in username
: # don't strictly check if it's email address or not
196 ep
, method
= 'user/email', 'email'
198 ep
, method
= 'oneTimePassword', 'userId'
200 login_response
= self
._download
_json
(
201 f
'https://api.abema.io/v1/auth/{ep}', None, note
='Logging in',
205 }).encode('utf-8'), headers
={
206 'Authorization': f
'bearer {self._get_device_token()}',
207 'Origin': 'https://abema.tv',
208 'Referer': 'https://abema.tv/',
209 'Content-Type': 'application/json',
212 AbemaTVBaseIE
._USERTOKEN
= login_response
['token']
213 self
._get
_media
_token
(True)
215 'device_id': AbemaTVBaseIE
._DEVICE
_ID
,
216 'usertoken': AbemaTVBaseIE
._USERTOKEN
,
218 self
.cache
.store(self
._NETRC
_MACHINE
, username
, auth_cache
)
220 def _call_api(self
, endpoint
, video_id
, query
=None, note
='Downloading JSON metadata'):
221 return self
._download
_json
(
222 f
'https://api.abema.io/{endpoint}', video_id
, query
=query
or {},
225 'Authorization': f
'bearer {self._get_device_token()}',
228 def _extract_breadcrumb_list(self
, webpage
, video_id
):
229 for jld
in re
.finditer(
230 r
'(?is)</span></li></ul><script[^>]+type=(["\']?
)application
/ld\
+json\
1[^
>]*>(?P
<json_ld
>.+?
)</script
>',
232 jsonld = self._parse_json(jld.group('json_ld
'), video_id, fatal=False)
233 if traverse_obj(jsonld, '@type') != 'BreadcrumbList
':
235 items = traverse_obj(jsonld, ('itemListElement
', ..., 'name
'))
241 class AbemaTVIE(AbemaTVBaseIE):
242 _VALID_URL = r'https?
://abema\
.tv
/(?P
<type>now
-on
-air|video
/episode|channels
/.+?
/slots
)/(?P
<id>[^?
/]+)'
244 'url
': 'https
://abema
.tv
/video
/episode
/194-25_s
2_p
1',
246 'id': '194-25_s
2_p
1',
247 'title
': '第
1話 「チーズケーキ」 「モーニング再び」
',
251 'episode
': '第
1話 「チーズケーキ」 「モーニング再び」
',
256 'url
': 'https
://abema
.tv
/channels
/anime
-live2
/slots
/E8tvAnMJ7a9a5d
',
258 'id': 'E8tvAnMJ7a9a5d
',
259 'title
': 'ゆるキャン△ SEASON2 全話一挙【無料ビデオ
72時間】
',
260 'series
': 'ゆるキャン△ SEASON2
',
261 'episode
': 'ゆるキャン△ SEASON2 全話一挙【無料ビデオ
72時間】
',
264 'description
': 'md5
:9c5a3172ae763278f9303922f0ea5b17
',
268 'url
': 'https
://abema
.tv
/video
/episode
/87-877_s
1282_p
31047',
270 'id': 'E8tvAnMJ7a9a5d
',
272 'description
': 'md5
:56d4fc1b4f7769ded5f923c55bb4695d
',
273 'thumbnail
': r're
:https
://hayabusa\
.io
/.+',
275 'episode
': '第
5話『光射す』
',
279 'url
': 'https
://abema
.tv
/now
-on
-air
/abema
-anime
',
283 # 'title
': '女子高生の無駄づかい 全話一挙【無料ビデオ
72時間】
',
284 'description
': 'md5
:55f2e61f46a17e9230802d7bcc913d5f
',
287 'skip
': 'Not supported until yt
-dlp implements native live downloader OR AbemaTV can start a local HTTP server
',
291 def _real_extract(self, url):
292 # starting download using infojson from this extractor is undefined behavior,
293 # and never be fixed in the future; you must trigger downloads by directly specifying URL.
294 # (unless there's a way to hook before downloading by extractor
)
295 video_id
, video_type
= self
._match
_valid
_url
(url
).group('id', 'type')
297 'Authorization': 'Bearer ' + self
._get
_device
_token
(),
299 video_type
= video_type
.split('/')[-1]
301 webpage
= self
._download
_webpage
(url
, video_id
)
302 canonical_url
= self
._search
_regex
(
303 r
'<link\s+rel="canonical"\s*href="(.+?)"', webpage
, 'canonical URL',
305 info
= self
._search
_json
_ld
(webpage
, video_id
, default
={})
307 title
= self
._search
_regex
(
308 r
'<span\s*class=".+?EpisodeTitleBlock__title">(.+?)</span>', webpage
, 'title', default
=None)
311 for jld
in re
.finditer(
312 r
'(?is)<span\s*class="com-m-Thumbnail__image">(?:</span>)?<script[^>]+type=(["\']?
)application
/ld\
+json\
1[^
>]*>(?P
<json_ld
>.+?
)</script
>',
314 jsonld = self._parse_json(jld.group('json_ld
'), video_id, fatal=False)
318 title = jsonld.get('caption
')
319 if not title and video_type == 'now
-on
-air
':
320 if not self._TIMETABLE:
321 # cache the timetable because it goes to 5MiB in size (!!)
322 self._TIMETABLE = self._download_json(
323 'https
://api
.abema
.io
/v1
/timetable
/dataSet?debug
=false
', video_id,
325 now = time_seconds(hours=9)
326 for slot in self._TIMETABLE.get('slots
', []):
327 if slot.get('channelId
') != video_id:
329 if slot['startAt
'] <= now and now < slot['endAt
']:
330 title = slot['title
']
333 # read breadcrumb on top of page
334 breadcrumb = self._extract_breadcrumb_list(webpage, video_id)
336 # breadcrumb list translates to: (e.g. 1st test for this IE)
337 # Home > Anime (genre) > Isekai Shokudo 2 (series name) > Episode 1 "Cheese cakes" "Morning again" (episode title)
339 info['series
'] = breadcrumb[-2]
340 info['episode
'] = breadcrumb[-1]
342 title = info['episode
']
344 description = self._html_search_regex(
345 (r'<p\s
+class="com-video-EpisodeDetailsBlock__content"><span\s
+class=".+?">(.+?
)</span
></p
><div
',
346 r'<span\s
+class=".+?SlotSummary.+?">(.+?
)</span
></div
><div
',),
347 webpage, 'description
', default=None, group=1)
349 og_desc = self._html_search_meta(
350 ('description
', 'og
:description
', 'twitter
:description
'), webpage)
352 description = re.sub(r'''(?sx)
354 アニメの動画を無料で見るならABEMA!| # anime
355 等、.+ # applies for most of categories
359 # canonical URL may contain season and episode number
360 mobj = re.search(r's(\d
+)_p(\d
+)$
', canonical_url)
362 seri = int_or_none(mobj.group(1), default=float('inf
'))
363 epis = int_or_none(mobj.group(2), default=float('inf
'))
364 info['season_number
'] = seri if seri < 100 else None
365 # some anime like Detective Conan (though not available in AbemaTV)
366 # has more than 1000 episodes (1026 as of 2021/11/15)
367 info['episode_number
'] = epis if epis < 2000 else None
369 is_live, m3u8_url = False, None
370 if video_type == 'now
-on
-air
':
372 channel_url = 'https
://api
.abema
.io
/v1
/channels
'
373 if video_id == 'news
-global':
374 channel_url = update_url_query(channel_url, {'division': '1'})
375 onair_channels = self._download_json(channel_url, video_id)
376 for ch in onair_channels['channels
']:
377 if video_id == ch['id']:
378 m3u8_url = ch['playback
']['hls
']
381 raise ExtractorError(f'Cannot find on
-air {video_id} channel
.', expected=True)
382 elif video_type == 'episode
':
383 api_response = self._download_json(
384 f'https
://api
.abema
.io
/v1
/video
/programs
/{video_id}
', video_id,
385 note='Checking playability
',
387 ondemand_types = traverse_obj(api_response, ('terms
', ..., 'onDemandType
'))
388 if 3 not in ondemand_types:
389 # cannot acquire decryption key for these streams
390 self.report_warning('This
is a premium
-only stream
')
391 info.update(traverse_obj(api_response, {
392 'series
': ('series
', 'title
'),
393 'season
': ('season
', 'name
'),
394 'season_number
': ('season
', 'sequence
'),
395 'episode_number
': ('episode
', 'number
'),
398 title = traverse_obj(api_response, ('episode
', 'title
'))
400 description = traverse_obj(api_response, ('episode
', 'content
'))
402 m3u8_url = f'https
://vod
-abematv
.akamaized
.net
/program
/{video_id}
/playlist
.m3u8
'
403 elif video_type == 'slots
':
404 api_response = self._download_json(
405 f'https
://api
.abema
.io
/v1
/media
/slots
/{video_id}
', video_id,
406 note='Checking playability
',
408 if not traverse_obj(api_response, ('slot
', 'flags
', 'timeshiftFree
'), default=False):
409 self.report_warning('This
is a premium
-only stream
')
411 m3u8_url = f'https
://vod
-abematv
.akamaized
.net
/slot
/{video_id}
/playlist
.m3u8
'
413 raise ExtractorError('Unreachable
')
416 self.report_warning("This is a livestream; yt-dlp doesn't support downloading natively
, but FFmpeg cannot handle m3u8 manifests
from AbemaTV
")
417 self.report_warning('Please consider using Streamlink to download these streams (https://github.com/streamlink/streamlink)')
418 formats = self._extract_m3u8_formats(
419 m3u8_url, video_id, ext='mp4', live=is_live)
424 'description': description,
431 class AbemaTVTitleIE(AbemaTVBaseIE):
432 _VALID_URL = r'https?://abema\.tv/video/title/(?P<id>[^?/]+)'
436 'url': 'https://abema.tv/video/title/90-1597',
439 'title': 'シャッフルアイランド',
441 'playlist_mincount': 2,
443 'url': 'https://abema.tv/video/title/193-132',
446 'title': '真心が届く~僕とスターのオフィス・ラブ!?~',
448 'playlist_mincount': 16,
450 'url': 'https://abema.tv/video/title/25-102',
453 'title': 'ソードアート・オンライン アリシゼーション',
455 'playlist_mincount': 24,
458 def _fetch_page(self, playlist_id, series_version, page):
459 programs = self._call_api(
460 f'v1/video/series/{playlist_id}/programs', playlist_id,
461 note=f'Downloading page {page + 1}',
463 'seriesVersion': series_version,
464 'offset': str(page * self._PAGE_SIZE),
466 'limit': str(self._PAGE_SIZE),
469 self.url_result(f'https://abema.tv/video/episode/{x}')
470 for x in traverse_obj(programs, ('programs', ..., 'id')))
472 def _entries(self, playlist_id, series_version):
473 return OnDemandPagedList(
474 functools.partial(self._fetch_page, playlist_id, series_version),
477 def _real_extract(self, url):
478 playlist_id = self._match_id(url)
479 series_info = self._call_api(f'v1/video/series/{playlist_id}', playlist_id)
481 return self.playlist_result(
482 self._entries(playlist_id, series_info['version']), playlist_id=playlist_id,
483 playlist_title=series_info.get('title'),
484 playlist_description=series_info.get('content'))