yt_dlp/extractor/adn.py

   1 import base64
   2 import binascii
   3 import json
   4 import os
   5 import random
   6 import time
   7
   8 from .common import InfoExtractor
   9 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
  10 from ..networking.exceptions import HTTPError
  11 from ..utils import (
  12     ExtractorError,
  13     ass_subtitles_timecode,
  14     bytes_to_intlist,
  15     bytes_to_long,
  16     float_or_none,
  17     int_or_none,
  18     intlist_to_bytes,
  19     long_to_bytes,
  20     parse_iso8601,
  21     pkcs1pad,
  22     str_or_none,
  23     strip_or_none,
  24     try_get,
  25     unified_strdate,
  26     urlencode_postdata,
  27 )
  28 from ..utils.traversal import traverse_obj
  29
  30
  31 class ADNBaseIE(InfoExtractor):
  32     IE_DESC = 'Animation Digital Network'
  33     _NETRC_MACHINE = 'animationdigitalnetwork'
  34     _BASE = 'animationdigitalnetwork.fr'
  35     _API_BASE_URL = f'https://gw.api.{_BASE}/'
  36     _PLAYER_BASE_URL = f'{_API_BASE_URL}player/'
  37     _HEADERS = {}
  38     _LOGIN_ERR_MESSAGE = 'Unable to log in'
  39     _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
  40     _POS_ALIGN_MAP = {
  41         'start': 1,
  42         'end': 3,
  43     }
  44     _LINE_ALIGN_MAP = {
  45         'middle': 8,
  46         'end': 4,
  47     }
  48
  49
  50 class ADNIE(ADNBaseIE):
  51     _VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.(?P<lang>fr|de)/video/[^/?#]+/(?P<id>\d+)'
  52     _TESTS = [{
  53         'url': 'https://animationdigitalnetwork.fr/video/fruits-basket/9841-episode-1-a-ce-soir',
  54         'md5': '1c9ef066ceb302c86f80c2b371615261',
  55         'info_dict': {
  56             'id': '9841',
  57             'ext': 'mp4',
  58             'title': 'Fruits Basket - Episode 1',
  59             'description': 'md5:14be2f72c3c96809b0ca424b0097d336',
  60             'series': 'Fruits Basket',
  61             'duration': 1437,
  62             'release_date': '20190405',
  63             'comment_count': int,
  64             'average_rating': float,
  65             'season_number': 1,
  66             'episode': 'À ce soir !',
  67             'episode_number': 1,
  68             'thumbnail': str,
  69             'season': 'Season 1',
  70         },
  71         'skip': 'Only available in French and German speaking Europe',
  72     }, {
  73         'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
  74         'only_matching': True,
  75     }, {
  76         'url': 'https://animationdigitalnetwork.de/video/the-eminence-in-shadow/23550-folge-1',
  77         'md5': '5c5651bf5791fa6fcd7906012b9d94e8',
  78         'info_dict': {
  79             'id': '23550',
  80             'ext': 'mp4',
  81             'episode_number': 1,
  82             'duration': 1417,
  83             'release_date': '20231004',
  84             'series': 'The Eminence in Shadow',
  85             'season_number': 2,
  86             'episode': str,
  87             'title': str,
  88             'thumbnail': str,
  89             'season': 'Season 2',
  90             'comment_count': int,
  91             'average_rating': float,
  92             'description': str,
  93         },
  94         # 'skip': 'Only available in French and German speaking Europe',
  95     }]
  96
  97     def _get_subtitles(self, sub_url, video_id):
  98         if not sub_url:
  99             return None
 100
 101         enc_subtitles = self._download_webpage(
 102             sub_url, video_id, 'Downloading subtitles location', fatal=False) or '{}'
 103         subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
 104         if subtitle_location:
 105             enc_subtitles = self._download_webpage(
 106                 subtitle_location, video_id, 'Downloading subtitles data',
 107                 fatal=False, headers={'Origin': 'https://' + self._BASE})
 108         if not enc_subtitles:
 109             return None
 110
 111         # http://animationdigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
 112         dec_subtitles = unpad_pkcs7(aes_cbc_decrypt_bytes(
 113             base64.b64decode(enc_subtitles[24:]),
 114             binascii.unhexlify(self._K + '7fac1178830cfe0c'),
 115             base64.b64decode(enc_subtitles[:24])))
 116         subtitles_json = self._parse_json(dec_subtitles.decode(), None, fatal=False)
 117         if not subtitles_json:
 118             return None
 119
 120         subtitles = {}
 121         for sub_lang, sub in subtitles_json.items():
 122             ssa = '''[Script Info]
 123 ScriptType:V4.00
 124 [V4 Styles]
 125 Format: Name,Fontname,Fontsize,PrimaryColour,SecondaryColour,TertiaryColour,BackColour,Bold,Italic,BorderStyle,Outline,Shadow,Alignment,MarginL,MarginR,MarginV,AlphaLevel,Encoding
 126 Style: Default,Arial,18,16777215,16777215,16777215,0,-1,0,1,1,0,2,20,20,20,0,0
 127 [Events]
 128 Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 129             for current in sub:
 130                 start, end, text, line_align, position_align = (
 131                     float_or_none(current.get('startTime')),
 132                     float_or_none(current.get('endTime')),
 133                     current.get('text'), current.get('lineAlign'),
 134                     current.get('positionAlign'))
 135                 if start is None or end is None or text is None:
 136                     continue
 137                 alignment = self._POS_ALIGN_MAP.get(position_align, 2) + self._LINE_ALIGN_MAP.get(line_align, 0)
 138                 ssa += os.linesep + 'Dialogue: Marked=0,{},{},Default,,0,0,0,,{}{}'.format(
 139                     ass_subtitles_timecode(start),
 140                     ass_subtitles_timecode(end),
 141                     '{\\a%d}' % alignment if alignment != 2 else '',
 142                     text.replace('\n', '\\N').replace('<i>', '{\\i1}').replace('</i>', '{\\i0}'))
 143
 144             if sub_lang == 'vostf':
 145                 sub_lang = 'fr'
 146             elif sub_lang == 'vostde':
 147                 sub_lang = 'de'
 148             subtitles.setdefault(sub_lang, []).extend([{
 149                 'ext': 'json',
 150                 'data': json.dumps(sub),
 151             }, {
 152                 'ext': 'ssa',
 153                 'data': ssa,
 154             }])
 155         return subtitles
 156
 157     def _perform_login(self, username, password):
 158         try:
 159             access_token = (self._download_json(
 160                 self._API_BASE_URL + 'authentication/login', None,
 161                 'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
 162                 data=urlencode_postdata({
 163                     'password': password,
 164                     'rememberMe': False,
 165                     'source': 'Web',
 166                     'username': username,
 167                 })) or {}).get('accessToken')
 168             if access_token:
 169                 self._HEADERS = {'authorization': 'Bearer ' + access_token}
 170         except ExtractorError as e:
 171             message = None
 172             if isinstance(e.cause, HTTPError) and e.cause.status == 401:
 173                 resp = self._parse_json(
 174                     e.cause.response.read().decode(), None, fatal=False) or {}
 175                 message = resp.get('message') or resp.get('code')
 176             self.report_warning(message or self._LOGIN_ERR_MESSAGE)
 177
 178     def _real_extract(self, url):
 179         lang, video_id = self._match_valid_url(url).group('lang', 'id')
 180         video_base_url = self._PLAYER_BASE_URL + f'video/{video_id}/'
 181         player = self._download_json(
 182             video_base_url + 'configuration', video_id,
 183             'Downloading player config JSON metadata',
 184             headers=self._HEADERS)['player']
 185         options = player['options']
 186
 187         user = options['user']
 188         if not user.get('hasAccess'):
 189             start_date = traverse_obj(options, ('video', 'startDate', {str}))
 190             if (parse_iso8601(start_date) or 0) > time.time():
 191                 raise ExtractorError(f'This video is not available yet. Release date: {start_date}', expected=True)
 192             self.raise_login_required('This video requires a subscription', method='password')
 193
 194         token = self._download_json(
 195             user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
 196             video_id, 'Downloading access token', headers={
 197                 'X-Player-Refresh-Token': user['refreshToken'],
 198             }, data=b'')['token']
 199
 200         links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
 201         self._K = ''.join(random.choices('0123456789abcdef', k=16))
 202         message = bytes_to_intlist(json.dumps({
 203             'k': self._K,
 204             't': token,
 205         }))
 206
 207         # Sometimes authentication fails for no good reason, retry with
 208         # a different random padding
 209         links_data = None
 210         for _ in range(3):
 211             padded_message = intlist_to_bytes(pkcs1pad(message, 128))
 212             n, e = self._RSA_KEY
 213             encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
 214             authorization = base64.b64encode(encrypted_message).decode()
 215
 216             try:
 217                 links_data = self._download_json(
 218                     links_url, video_id, 'Downloading links JSON metadata', headers={
 219                         'X-Player-Token': authorization,
 220                         'X-Target-Distribution': lang,
 221                         **self._HEADERS,
 222                     }, query={
 223                         'freeWithAds': 'true',
 224                         'adaptive': 'false',
 225                         'withMetadata': 'true',
 226                         'source': 'Web',
 227                     })
 228                 break
 229             except ExtractorError as e:
 230                 if not isinstance(e.cause, HTTPError):
 231                     raise e
 232
 233                 if e.cause.status == 401:
 234                     # This usually goes away with a different random pkcs1pad, so retry
 235                     continue
 236
 237                 error = self._parse_json(e.cause.response.read(), video_id)
 238                 message = error.get('message')
 239                 if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
 240                     self.raise_geo_restricted(msg=message)
 241                 raise ExtractorError(message)
 242         else:
 243             raise ExtractorError('Giving up retrying')
 244
 245         links = links_data.get('links') or {}
 246         metas = links_data.get('metadata') or {}
 247         sub_url = (links.get('subtitles') or {}).get('all')
 248         video_info = links_data.get('video') or {}
 249         title = metas['title']
 250
 251         formats = []
 252         for format_id, qualities in (links.get('streaming') or {}).items():
 253             if not isinstance(qualities, dict):
 254                 continue
 255             for quality, load_balancer_url in qualities.items():
 256                 load_balancer_data = self._download_json(
 257                     load_balancer_url, video_id,
 258                     f'Downloading {format_id} {quality} JSON metadata',
 259                     fatal=False) or {}
 260                 m3u8_url = load_balancer_data.get('location')
 261                 if not m3u8_url:
 262                     continue
 263                 m3u8_formats = self._extract_m3u8_formats(
 264                     m3u8_url, video_id, 'mp4', 'm3u8_native',
 265                     m3u8_id=format_id, fatal=False)
 266                 if format_id == 'vf':
 267                     for f in m3u8_formats:
 268                         f['language'] = 'fr'
 269                 elif format_id == 'vde':
 270                     for f in m3u8_formats:
 271                         f['language'] = 'de'
 272                 formats.extend(m3u8_formats)
 273
 274         if not formats:
 275             self.raise_login_required('This video requires a subscription', method='password')
 276
 277         video = (self._download_json(
 278             self._API_BASE_URL + f'video/{video_id}', video_id,
 279             'Downloading additional video metadata', fatal=False) or {}).get('video') or {}
 280         show = video.get('show') or {}
 281
 282         return {
 283             'id': video_id,
 284             'title': title,
 285             'description': strip_or_none(metas.get('summary') or video.get('summary')),
 286             'thumbnail': video_info.get('image') or player.get('image'),
 287             'formats': formats,
 288             'subtitles': self.extract_subtitles(sub_url, video_id),
 289             'episode': metas.get('subtitle') or video.get('name'),
 290             'episode_number': int_or_none(video.get('shortNumber')),
 291             'series': show.get('title'),
 292             'season_number': int_or_none(video.get('season')),
 293             'duration': int_or_none(video_info.get('duration') or video.get('duration')),
 294             'release_date': unified_strdate(video.get('releaseDate')),
 295             'average_rating': float_or_none(video.get('rating') or metas.get('rating')),
 296             'comment_count': int_or_none(video.get('commentsCount')),
 297         }
 298
 299
 300 class ADNSeasonIE(ADNBaseIE):
 301     _VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.(?P<lang>fr|de)/video/(?P<id>[^/?#]+)/?(?:$|[#?])'
 302     _TESTS = [{
 303         'url': 'https://animationdigitalnetwork.fr/video/tokyo-mew-mew-new',
 304         'playlist_count': 12,
 305         'info_dict': {
 306             'id': '911',
 307             'title': 'Tokyo Mew Mew New',
 308         },
 309         # 'skip': 'Only available in French end German speaking Europe',
 310     }]
 311
 312     def _real_extract(self, url):
 313         lang, video_show_slug = self._match_valid_url(url).group('lang', 'id')
 314         show = self._download_json(
 315             f'{self._API_BASE_URL}show/{video_show_slug}/', video_show_slug,
 316             'Downloading show JSON metadata', headers=self._HEADERS)['show']
 317         show_id = str(show['id'])
 318         episodes = self._download_json(
 319             f'{self._API_BASE_URL}video/show/{show_id}', video_show_slug,
 320             'Downloading episode list', headers={
 321                 'X-Target-Distribution': lang,
 322                 **self._HEADERS,
 323             }, query={
 324                 'order': 'asc',
 325                 'limit': '-1',
 326             })
 327
 328         def entries():
 329             for episode_id in traverse_obj(episodes, ('videos', ..., 'id', {str_or_none})):
 330                 yield self.url_result(
 331                     f'https://animationdigitalnetwork.{lang}/video/{video_show_slug}/{episode_id}',
 332                     ADNIE, episode_id)
 333
 334         return self.playlist_result(entries(), show_id, show.get('title'))