yt_dlp/extractor/adn.py

   1 import base64
   2 import binascii
   3 import json
   4 import os
   5 import random
   6
   7 from .common import InfoExtractor
   8 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
   9 from ..compat import compat_b64decode
  10 from ..networking.exceptions import HTTPError
  11 from ..utils import (
  12     ass_subtitles_timecode,
  13     bytes_to_intlist,
  14     bytes_to_long,
  15     ExtractorError,
  16     float_or_none,
  17     int_or_none,
  18     intlist_to_bytes,
  19     long_to_bytes,
  20     pkcs1pad,
  21     strip_or_none,
  22     try_get,
  23     unified_strdate,
  24     urlencode_postdata,
  25 )
  26
  27
  28 class ADNIE(InfoExtractor):
  29     IE_DESC = 'Animation Digital Network'
  30     _VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
  31     _TESTS = [{
  32         'url': 'https://animationdigitalnetwork.fr/video/fruits-basket/9841-episode-1-a-ce-soir',
  33         'md5': '1c9ef066ceb302c86f80c2b371615261',
  34         'info_dict': {
  35             'id': '9841',
  36             'ext': 'mp4',
  37             'title': 'Fruits Basket - Episode 1',
  38             'description': 'md5:14be2f72c3c96809b0ca424b0097d336',
  39             'series': 'Fruits Basket',
  40             'duration': 1437,
  41             'release_date': '20190405',
  42             'comment_count': int,
  43             'average_rating': float,
  44             'season_number': 1,
  45             'episode': 'À ce soir !',
  46             'episode_number': 1,
  47         },
  48         'skip': 'Only available in region (FR, ...)',
  49     }, {
  50         'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
  51         'only_matching': True,
  52     }]
  53
  54     _NETRC_MACHINE = 'animationdigitalnetwork'
  55     _BASE = 'animationdigitalnetwork.fr'
  56     _API_BASE_URL = 'https://gw.api.' + _BASE + '/'
  57     _PLAYER_BASE_URL = _API_BASE_URL + 'player/'
  58     _HEADERS = {}
  59     _LOGIN_ERR_MESSAGE = 'Unable to log in'
  60     _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
  61     _POS_ALIGN_MAP = {
  62         'start': 1,
  63         'end': 3,
  64     }
  65     _LINE_ALIGN_MAP = {
  66         'middle': 8,
  67         'end': 4,
  68     }
  69
  70     def _get_subtitles(self, sub_url, video_id):
  71         if not sub_url:
  72             return None
  73
  74         enc_subtitles = self._download_webpage(
  75             sub_url, video_id, 'Downloading subtitles location', fatal=False) or '{}'
  76         subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
  77         if subtitle_location:
  78             enc_subtitles = self._download_webpage(
  79                 subtitle_location, video_id, 'Downloading subtitles data',
  80                 fatal=False, headers={'Origin': 'https://' + self._BASE})
  81         if not enc_subtitles:
  82             return None
  83
  84         # http://animationdigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
  85         dec_subtitles = unpad_pkcs7(aes_cbc_decrypt_bytes(
  86             compat_b64decode(enc_subtitles[24:]),
  87             binascii.unhexlify(self._K + '7fac1178830cfe0c'),
  88             compat_b64decode(enc_subtitles[:24])))
  89         subtitles_json = self._parse_json(dec_subtitles.decode(), None, fatal=False)
  90         if not subtitles_json:
  91             return None
  92
  93         subtitles = {}
  94         for sub_lang, sub in subtitles_json.items():
  95             ssa = '''[Script Info]
  96 ScriptType:V4.00
  97 [V4 Styles]
  98 Format: Name,Fontname,Fontsize,PrimaryColour,SecondaryColour,TertiaryColour,BackColour,Bold,Italic,BorderStyle,Outline,Shadow,Alignment,MarginL,MarginR,MarginV,AlphaLevel,Encoding
  99 Style: Default,Arial,18,16777215,16777215,16777215,0,-1,0,1,1,0,2,20,20,20,0,0
 100 [Events]
 101 Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 102             for current in sub:
 103                 start, end, text, line_align, position_align = (
 104                     float_or_none(current.get('startTime')),
 105                     float_or_none(current.get('endTime')),
 106                     current.get('text'), current.get('lineAlign'),
 107                     current.get('positionAlign'))
 108                 if start is None or end is None or text is None:
 109                     continue
 110                 alignment = self._POS_ALIGN_MAP.get(position_align, 2) + self._LINE_ALIGN_MAP.get(line_align, 0)
 111                 ssa += os.linesep + 'Dialogue: Marked=0,%s,%s,Default,,0,0,0,,%s%s' % (
 112                     ass_subtitles_timecode(start),
 113                     ass_subtitles_timecode(end),
 114                     '{\\a%d}' % alignment if alignment != 2 else '',
 115                     text.replace('\n', '\\N').replace('<i>', '{\\i1}').replace('</i>', '{\\i0}'))
 116
 117             if sub_lang == 'vostf':
 118                 sub_lang = 'fr'
 119             subtitles.setdefault(sub_lang, []).extend([{
 120                 'ext': 'json',
 121                 'data': json.dumps(sub),
 122             }, {
 123                 'ext': 'ssa',
 124                 'data': ssa,
 125             }])
 126         return subtitles
 127
 128     def _perform_login(self, username, password):
 129         try:
 130             access_token = (self._download_json(
 131                 self._API_BASE_URL + 'authentication/login', None,
 132                 'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
 133                 data=urlencode_postdata({
 134                     'password': password,
 135                     'rememberMe': False,
 136                     'source': 'Web',
 137                     'username': username,
 138                 })) or {}).get('accessToken')
 139             if access_token:
 140                 self._HEADERS = {'authorization': 'Bearer ' + access_token}
 141         except ExtractorError as e:
 142             message = None
 143             if isinstance(e.cause, HTTPError) and e.cause.status == 401:
 144                 resp = self._parse_json(
 145                     e.cause.response.read().decode(), None, fatal=False) or {}
 146                 message = resp.get('message') or resp.get('code')
 147             self.report_warning(message or self._LOGIN_ERR_MESSAGE)
 148
 149     def _real_extract(self, url):
 150         video_id = self._match_id(url)
 151         video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id
 152         player = self._download_json(
 153             video_base_url + 'configuration', video_id,
 154             'Downloading player config JSON metadata',
 155             headers=self._HEADERS)['player']
 156         options = player['options']
 157
 158         user = options['user']
 159         if not user.get('hasAccess'):
 160             self.raise_login_required()
 161
 162         token = self._download_json(
 163             user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
 164             video_id, 'Downloading access token', headers={
 165                 'x-player-refresh-token': user['refreshToken']
 166             }, data=b'')['token']
 167
 168         links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
 169         self._K = ''.join(random.choices('0123456789abcdef', k=16))
 170         message = bytes_to_intlist(json.dumps({
 171             'k': self._K,
 172             't': token,
 173         }))
 174
 175         # Sometimes authentication fails for no good reason, retry with
 176         # a different random padding
 177         links_data = None
 178         for _ in range(3):
 179             padded_message = intlist_to_bytes(pkcs1pad(message, 128))
 180             n, e = self._RSA_KEY
 181             encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
 182             authorization = base64.b64encode(encrypted_message).decode()
 183
 184             try:
 185                 links_data = self._download_json(
 186                     links_url, video_id, 'Downloading links JSON metadata', headers={
 187                         'X-Player-Token': authorization
 188                     }, query={
 189                         'freeWithAds': 'true',
 190                         'adaptive': 'false',
 191                         'withMetadata': 'true',
 192                         'source': 'Web'
 193                     })
 194                 break
 195             except ExtractorError as e:
 196                 if not isinstance(e.cause, HTTPError):
 197                     raise e
 198
 199                 if e.cause.status == 401:
 200                     # This usually goes away with a different random pkcs1pad, so retry
 201                     continue
 202
 203                 error = self._parse_json(e.cause.response.read(), video_id)
 204                 message = error.get('message')
 205                 if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
 206                     self.raise_geo_restricted(msg=message)
 207                 raise ExtractorError(message)
 208         else:
 209             raise ExtractorError('Giving up retrying')
 210
 211         links = links_data.get('links') or {}
 212         metas = links_data.get('metadata') or {}
 213         sub_url = (links.get('subtitles') or {}).get('all')
 214         video_info = links_data.get('video') or {}
 215         title = metas['title']
 216
 217         formats = []
 218         for format_id, qualities in (links.get('streaming') or {}).items():
 219             if not isinstance(qualities, dict):
 220                 continue
 221             for quality, load_balancer_url in qualities.items():
 222                 load_balancer_data = self._download_json(
 223                     load_balancer_url, video_id,
 224                     'Downloading %s %s JSON metadata' % (format_id, quality),
 225                     fatal=False) or {}
 226                 m3u8_url = load_balancer_data.get('location')
 227                 if not m3u8_url:
 228                     continue
 229                 m3u8_formats = self._extract_m3u8_formats(
 230                     m3u8_url, video_id, 'mp4', 'm3u8_native',
 231                     m3u8_id=format_id, fatal=False)
 232                 if format_id == 'vf':
 233                     for f in m3u8_formats:
 234                         f['language'] = 'fr'
 235                 formats.extend(m3u8_formats)
 236
 237         video = (self._download_json(
 238             self._API_BASE_URL + 'video/%s' % video_id, video_id,
 239             'Downloading additional video metadata', fatal=False) or {}).get('video') or {}
 240         show = video.get('show') or {}
 241
 242         return {
 243             'id': video_id,
 244             'title': title,
 245             'description': strip_or_none(metas.get('summary') or video.get('summary')),
 246             'thumbnail': video_info.get('image') or player.get('image'),
 247             'formats': formats,
 248             'subtitles': self.extract_subtitles(sub_url, video_id),
 249             'episode': metas.get('subtitle') or video.get('name'),
 250             'episode_number': int_or_none(video.get('shortNumber')),
 251             'series': show.get('title'),
 252             'season_number': int_or_none(video.get('season')),
 253             'duration': int_or_none(video_info.get('duration') or video.get('duration')),
 254             'release_date': unified_strdate(video.get('releaseDate')),
 255             'average_rating': float_or_none(video.get('rating') or metas.get('rating')),
 256             'comment_count': int_or_none(video.get('commentsCount')),
 257         }