yt_dlp/extractor/wrestleuniverse.py

   1 import base64
   2 import binascii
   3 import json
   4 import time
   5
   6 from .common import InfoExtractor
   7 from ..dependencies import Cryptodome
   8 from ..utils import (
   9     ExtractorError,
  10     int_or_none,
  11     jwt_decode_hs256,
  12     traverse_obj,
  13     try_call,
  14     url_or_none,
  15 )
  16
  17
  18 class WrestleUniverseBaseIE(InfoExtractor):
  19     _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
  20     _API_PATH = None
  21     _TOKEN = None
  22     _TOKEN_EXPIRY = None
  23
  24     def _get_token_cookie(self):
  25         if not self._TOKEN or not self._TOKEN_EXPIRY:
  26             self._TOKEN = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
  27             if not self._TOKEN:
  28                 self.raise_login_required()
  29             expiry = traverse_obj(jwt_decode_hs256(self._TOKEN), ('exp', {int_or_none}))
  30             if not expiry:
  31                 raise ExtractorError('There was a problem with the token cookie')
  32             self._TOKEN_EXPIRY = expiry
  33
  34         if self._TOKEN_EXPIRY <= int(time.time()):
  35             raise ExtractorError(
  36                 'Expired token. Refresh your cookies in browser and try again', expected=True)
  37
  38         return self._TOKEN
  39
  40     def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
  41         headers = {'CA-CID': ''}
  42         if data:
  43             headers['Content-Type'] = 'application/json;charset=utf-8'
  44             data = json.dumps(data, separators=(',', ':')).encode()
  45         if auth:
  46             headers['Authorization'] = f'Bearer {self._get_token_cookie()}'
  47         return self._download_json(
  48             f'https://api.wrestle-universe.com/v1/{self._API_PATH}/{video_id}{param}', video_id,
  49             note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
  50             data=data, headers=headers, query=query, fatal=fatal)
  51
  52     def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True):
  53         if not Cryptodome:
  54             raise ExtractorError('pycryptodomex not found. Please install', expected=True)
  55         private_key = Cryptodome.PublicKey.RSA.generate(2048)
  56         cipher = Cryptodome.Cipher.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.Hash.SHA1)
  57
  58         def decrypt(data):
  59             if not data:
  60                 return None
  61             try:
  62                 return cipher.decrypt(base64.b64decode(data)).decode()
  63             except (ValueError, binascii.Error) as e:
  64                 raise ExtractorError(f'Could not decrypt data: {e}')
  65
  66         token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
  67         api_json = self._call_api(video_id, param, msg, data={
  68             # 'deviceId' (random uuid4 generated at login) is not required yet
  69             'token': token,
  70             **data,
  71         }, query=query, fatal=fatal)
  72         return api_json, decrypt
  73
  74     def _download_metadata(self, url, video_id, lang, props_key):
  75         metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
  76         if not metadata:
  77             webpage = self._download_webpage(url, video_id)
  78             nextjs_data = self._search_nextjs_data(webpage, video_id)
  79             metadata = traverse_obj(nextjs_data, ('props', 'pageProps', props_key, {dict})) or {}
  80         return metadata
  81
  82     def _get_formats(self, data, path, video_id=None):
  83         hls_url = traverse_obj(data, path, get_all=False)
  84         if not hls_url and not data.get('canWatch'):
  85             self.raise_no_formats(
  86                 'This account does not have access to the requested content', expected=True)
  87         elif not hls_url:
  88             self.raise_no_formats('No supported formats found')
  89         return self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', live=True)
  90
  91
  92 class WrestleUniverseVODIE(WrestleUniverseBaseIE):
  93     _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'videos'
  94     _TESTS = [{
  95         'url': 'https://www.wrestle-universe.com/en/videos/dp8mpjmcKfxzUhEHM2uFws',
  96         'info_dict': {
  97             'id': 'dp8mpjmcKfxzUhEHM2uFws',
  98             'ext': 'mp4',
  99             'title': 'The 3rd “Futari wa Princess” Max Heart Tournament',
 100             'description': 'md5:318d5061e944797fbbb81d5c7dd00bf5',
 101             'location': '埼玉・春日部ふれあいキューブ',
 102             'channel': 'tjpw',
 103             'duration': 7119,
 104             'timestamp': 1674979200,
 105             'upload_date': '20230129',
 106             'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
 107             'chapters': 'count:7',
 108             'cast': 'count:18',
 109         },
 110         'params': {
 111             'skip_download': 'm3u8',
 112         },
 113     }]
 114
 115     _API_PATH = 'videoEpisodes'
 116
 117     def _real_extract(self, url):
 118         lang, video_id = self._match_valid_url(url).group('lang', 'id')
 119         metadata = self._download_metadata(url, video_id, lang, 'videoEpisodeFallbackData')
 120         video_data = self._call_api(video_id, ':watch', 'watch', data={
 121             # 'deviceId' is required if ignoreDeviceRestriction is False
 122             'ignoreDeviceRestriction': True,
 123         })
 124
 125         return {
 126             'id': video_id,
 127             'formats': self._get_formats(video_data, (
 128                 (('protocolHls', 'url'), ('chromecastUrls', ...)), {url_or_none}), video_id),
 129             **traverse_obj(metadata, {
 130                 'title': ('displayName', {str}),
 131                 'description': ('description', {str}),
 132                 'channel': ('labels', 'group', {str}),
 133                 'location': ('labels', 'venue', {str}),
 134                 'timestamp': ('watchStartTime', {int_or_none}),
 135                 'thumbnail': ('keyVisualUrl', {url_or_none}),
 136                 'cast': ('casts', ..., 'displayName', {str}),
 137                 'duration': ('duration', {int}),
 138                 'chapters': ('videoChapters', lambda _, v: isinstance(v.get('start'), int), {
 139                     'title': ('displayName', {str}),
 140                     'start_time': ('start', {int}),
 141                     'end_time': ('end', {int}),
 142                 }),
 143             }),
 144         }
 145
 146
 147 class WrestleUniversePPVIE(WrestleUniverseBaseIE):
 148     _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'lives'
 149     _TESTS = [{
 150         'note': 'HLS AES-128 key obtained via API',
 151         'url': 'https://www.wrestle-universe.com/en/lives/buH9ibbfhdJAY4GKZcEuJX',
 152         'info_dict': {
 153             'id': 'buH9ibbfhdJAY4GKZcEuJX',
 154             'ext': 'mp4',
 155             'title': '【PPV】Beyond the origins, into the future',
 156             'description': 'md5:9a872db68cd09be4a1e35a3ee8b0bdfc',
 157             'channel': 'tjpw',
 158             'location': '東京・Twin Box AKIHABARA',
 159             'duration': 10098,
 160             'timestamp': 1675076400,
 161             'upload_date': '20230130',
 162             'thumbnail': 'https://image.asset.wrestle-universe.com/rJs2m7cBaLXrwCcxMdQGRM/rJs2m7cBaLXrwCcxMdQGRM',
 163             'thumbnails': 'count:3',
 164             'hls_aes': {
 165                 'key': '5633184acd6e43f1f1ac71c6447a4186',
 166                 'iv': '5bac71beb33197d5600337ce86de7862',
 167             },
 168         },
 169         'params': {
 170             'skip_download': 'm3u8',
 171         },
 172     }, {
 173         'note': 'unencrypted HLS',
 174         'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
 175         'info_dict': {
 176             'id': 'wUG8hP5iApC63jbtQzhVVx',
 177             'ext': 'mp4',
 178             'title': 'GRAND PRINCESS \'22',
 179             'description': 'md5:e4f43d0d4262de3952ff34831bc99858',
 180             'channel': 'tjpw',
 181             'location': '東京・両国国技館',
 182             'duration': 18044,
 183             'timestamp': 1647665400,
 184             'upload_date': '20220319',
 185             'thumbnail': 'https://image.asset.wrestle-universe.com/i8jxSTCHPfdAKD4zN41Psx/i8jxSTCHPfdAKD4zN41Psx',
 186             'thumbnails': 'count:3',
 187         },
 188         'params': {
 189             'skip_download': 'm3u8',
 190         },
 191     }]
 192
 193     _API_PATH = 'events'
 194
 195     def _real_extract(self, url):
 196         lang, video_id = self._match_valid_url(url).group('lang', 'id')
 197         metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
 198
 199         info = traverse_obj(metadata, {
 200             'title': ('displayName', {str}),
 201             'description': ('description', {str}),
 202             'channel': ('labels', 'group', {str}),
 203             'location': ('labels', 'venue', {str}),
 204             'timestamp': ('startTime', {int_or_none}),
 205             'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
 206         })
 207
 208         ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
 209         if info.get('timestamp') and ended_time:
 210             info['duration'] = ended_time - info['timestamp']
 211
 212         video_data, decrypt = self._call_encrypted_api(
 213             video_id, ':watchArchive', 'watch archive', data={'method': 1})
 214         formats = self._get_formats(video_data, (
 215             ('hls', None), ('urls', 'chromecastUrls'), ..., {url_or_none}), video_id)
 216         for f in formats:
 217             # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
 218             if f.get('tbr'):
 219                 f['tbr'] = int(f['tbr'] / 2.5)
 220
 221         hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
 222         if not hls_aes_key and traverse_obj(video_data, ('hls', 'encryptType', {int}), default=0) > 0:
 223             self.report_warning('HLS AES-128 key was not found in API response')
 224
 225         return {
 226             'id': video_id,
 227             'formats': formats,
 228             'hls_aes': {
 229                 'key': hls_aes_key,
 230                 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
 231             },
 232             **info,
 233         }