yt_dlp/extractor/wrestleuniverse.py

   1 import base64
   2 import binascii
   3 import json
   4 import time
   5 import uuid
   6
   7 from .common import InfoExtractor
   8 from ..dependencies import Cryptodome
   9 from ..utils import (
  10     ExtractorError,
  11     int_or_none,
  12     jwt_decode_hs256,
  13     traverse_obj,
  14     try_call,
  15     url_or_none,
  16     urlencode_postdata,
  17 )
  18
  19
  20 class WrestleUniverseBaseIE(InfoExtractor):
  21     _NETRC_MACHINE = 'wrestleuniverse'
  22     _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
  23     _API_PATH = None
  24     _REAL_TOKEN = None
  25     _TOKEN_EXPIRY = None
  26     _REFRESH_TOKEN = None
  27     _DEVICE_ID = None
  28     _LOGIN_QUERY = {'key': 'AIzaSyCaRPBsDQYVDUWWBXjsTrHESi2r_F3RAdA'}
  29     _LOGIN_HEADERS = {
  30         'Accept': '*/*',
  31         'Content-Type': 'application/json',
  32         'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
  33         'X-Firebase-gmpid': '1:307308870738:web:820f38fe5150c8976e338b',
  34         'Referer': 'https://www.wrestle-universe.com/',
  35         'Origin': 'https://www.wrestle-universe.com',
  36     }
  37
  38     @property
  39     def _TOKEN(self):
  40         if not self._REAL_TOKEN or not self._TOKEN_EXPIRY:
  41             token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
  42             if not token and not self._REFRESH_TOKEN:
  43                 self.raise_login_required()
  44             self._TOKEN = token
  45
  46         if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
  47             if not self._REFRESH_TOKEN:
  48                 raise ExtractorError(
  49                     'Expired token. Refresh your cookies in browser and try again', expected=True)
  50             self._refresh_token()
  51
  52         return self._REAL_TOKEN
  53
  54     @_TOKEN.setter
  55     def _TOKEN(self, value):
  56         self._REAL_TOKEN = value
  57
  58         expiry = traverse_obj(value, ({jwt_decode_hs256}, 'exp', {int_or_none}))
  59         if not expiry:
  60             raise ExtractorError('There was a problem with the auth token')
  61         self._TOKEN_EXPIRY = expiry
  62
  63     def _perform_login(self, username, password):
  64         login = self._download_json(
  65             'https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword', None,
  66             'Logging in', query=self._LOGIN_QUERY, headers=self._LOGIN_HEADERS, data=json.dumps({
  67                 'returnSecureToken': True,
  68                 'email': username,
  69                 'password': password,
  70             }, separators=(',', ':')).encode())
  71         self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
  72         if not self._REFRESH_TOKEN:
  73             self.report_warning('No refresh token was granted')
  74         self._TOKEN = traverse_obj(login, ('idToken', {str}))
  75
  76     def _real_initialize(self):
  77         if WrestleUniverseBaseIE._DEVICE_ID:
  78             return
  79
  80         WrestleUniverseBaseIE._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key='WrestleUniverse')[0]
  81         if not WrestleUniverseBaseIE._DEVICE_ID:
  82             WrestleUniverseBaseIE._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
  83             if WrestleUniverseBaseIE._DEVICE_ID:
  84                 return
  85             WrestleUniverseBaseIE._DEVICE_ID = str(uuid.uuid4())
  86
  87         self.cache.store(self._NETRC_MACHINE, 'device_id', WrestleUniverseBaseIE._DEVICE_ID)
  88
  89     def _refresh_token(self):
  90         refresh = self._download_json(
  91             'https://securetoken.googleapis.com/v1/token', None, 'Refreshing token',
  92             query=self._LOGIN_QUERY, data=urlencode_postdata({
  93                 'grant_type': 'refresh_token',
  94                 'refresh_token': self._REFRESH_TOKEN,
  95             }), headers={
  96                 **self._LOGIN_HEADERS,
  97                 'Content-Type': 'application/x-www-form-urlencoded',
  98             })
  99         if traverse_obj(refresh, ('refresh_token', {str})):
 100             self._REFRESH_TOKEN = refresh['refresh_token']
 101         token = traverse_obj(refresh, 'access_token', 'id_token', expected_type=str)
 102         if not token:
 103             raise ExtractorError('No auth token returned from refresh request')
 104         self._TOKEN = token
 105
 106     def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
 107         headers = {'CA-CID': ''}
 108         if data:
 109             headers['Content-Type'] = 'application/json;charset=utf-8'
 110             data = json.dumps(data, separators=(',', ':')).encode()
 111         if auth:
 112             headers['Authorization'] = f'Bearer {self._TOKEN}'
 113         return self._download_json(
 114             f'https://api.wrestle-universe.com/v1/{self._API_PATH}/{video_id}{param}', video_id,
 115             note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
 116             data=data, headers=headers, query=query, fatal=fatal)
 117
 118     def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True):
 119         if not Cryptodome.RSA:
 120             raise ExtractorError('pycryptodomex not found. Please install', expected=True)
 121         private_key = Cryptodome.RSA.generate(2048)
 122         cipher = Cryptodome.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.SHA1)
 123
 124         def decrypt(data):
 125             if not data:
 126                 return None
 127             try:
 128                 return cipher.decrypt(base64.b64decode(data)).decode()
 129             except (ValueError, binascii.Error) as e:
 130                 raise ExtractorError(f'Could not decrypt data: {e}')
 131
 132         token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
 133         api_json = self._call_api(video_id, param, msg, data={
 134             'deviceId': self._DEVICE_ID,
 135             'token': token,
 136             **data,
 137         }, query=query, fatal=fatal)
 138         return api_json, decrypt
 139
 140     def _download_metadata(self, url, video_id, lang, props_key):
 141         metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
 142         if not metadata:
 143             webpage = self._download_webpage(url, video_id)
 144             nextjs_data = self._search_nextjs_data(webpage, video_id)
 145             metadata = traverse_obj(nextjs_data, ('props', 'pageProps', props_key, {dict})) or {}
 146         return metadata
 147
 148     def _get_formats(self, data, path, video_id=None):
 149         hls_url = traverse_obj(data, path, get_all=False)
 150         if not hls_url and not data.get('canWatch'):
 151             self.raise_no_formats(
 152                 'This account does not have access to the requested content', expected=True)
 153         elif not hls_url:
 154             self.raise_no_formats('No supported formats found')
 155         return self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', live=True)
 156
 157
 158 class WrestleUniverseVODIE(WrestleUniverseBaseIE):
 159     _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'videos'
 160     _TESTS = [{
 161         'url': 'https://www.wrestle-universe.com/en/videos/dp8mpjmcKfxzUhEHM2uFws',
 162         'info_dict': {
 163             'id': 'dp8mpjmcKfxzUhEHM2uFws',
 164             'ext': 'mp4',
 165             'title': 'The 3rd “Futari wa Princess” Max Heart Tournament',
 166             'description': 'md5:318d5061e944797fbbb81d5c7dd00bf5',
 167             'location': '埼玉・春日部ふれあいキューブ',
 168             'channel': 'tjpw',
 169             'duration': 7119,
 170             'timestamp': 1674979200,
 171             'upload_date': '20230129',
 172             'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
 173             'chapters': 'count:7',
 174             'cast': 'count:21',
 175         },
 176         'params': {
 177             'skip_download': 'm3u8',
 178         },
 179     }]
 180
 181     _API_PATH = 'videoEpisodes'
 182
 183     def _real_extract(self, url):
 184         lang, video_id = self._match_valid_url(url).group('lang', 'id')
 185         metadata = self._download_metadata(url, video_id, lang, 'videoEpisodeFallbackData')
 186         video_data = self._call_api(video_id, ':watch', 'watch', data={
 187             # 'deviceId' is required if ignoreDeviceRestriction is False
 188             'ignoreDeviceRestriction': True,
 189         })
 190
 191         return {
 192             'id': video_id,
 193             'formats': self._get_formats(video_data, (
 194                 (('protocolHls', 'url'), ('chromecastUrls', ...)), {url_or_none}), video_id),
 195             **traverse_obj(metadata, {
 196                 'title': ('displayName', {str}),
 197                 'description': ('description', {str}),
 198                 'channel': ('labels', 'group', {str}),
 199                 'location': ('labels', 'venue', {str}),
 200                 'timestamp': ('watchStartTime', {int_or_none}),
 201                 'thumbnail': ('keyVisualUrl', {url_or_none}),
 202                 'cast': ('casts', ..., 'displayName', {str}),
 203                 'duration': ('duration', {int}),
 204                 'chapters': ('videoChapters', lambda _, v: isinstance(v.get('start'), int), {
 205                     'title': ('displayName', {str}),
 206                     'start_time': ('start', {int}),
 207                     'end_time': ('end', {int}),
 208                 }),
 209             }),
 210         }
 211
 212
 213 class WrestleUniversePPVIE(WrestleUniverseBaseIE):
 214     _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'lives'
 215     _TESTS = [{
 216         'note': 'HLS AES-128 key obtained via API',
 217         'url': 'https://www.wrestle-universe.com/en/lives/buH9ibbfhdJAY4GKZcEuJX',
 218         'info_dict': {
 219             'id': 'buH9ibbfhdJAY4GKZcEuJX',
 220             'ext': 'mp4',
 221             'title': '【PPV】Beyond the origins, into the future',
 222             'description': 'md5:9a872db68cd09be4a1e35a3ee8b0bdfc',
 223             'channel': 'tjpw',
 224             'location': '東京・Twin Box AKIHABARA',
 225             'duration': 10098,
 226             'timestamp': 1675076400,
 227             'upload_date': '20230130',
 228             'thumbnail': 'https://image.asset.wrestle-universe.com/rJs2m7cBaLXrwCcxMdQGRM/rJs2m7cBaLXrwCcxMdQGRM',
 229             'thumbnails': 'count:3',
 230             'hls_aes': {
 231                 'key': '5633184acd6e43f1f1ac71c6447a4186',
 232                 'iv': '5bac71beb33197d5600337ce86de7862',
 233             },
 234         },
 235         'params': {
 236             'skip_download': 'm3u8',
 237         },
 238         'skip': 'No longer available',
 239     }, {
 240         'note': 'unencrypted HLS',
 241         'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
 242         'info_dict': {
 243             'id': 'wUG8hP5iApC63jbtQzhVVx',
 244             'ext': 'mp4',
 245             'title': 'GRAND PRINCESS \'22',
 246             'description': 'md5:e4f43d0d4262de3952ff34831bc99858',
 247             'channel': 'tjpw',
 248             'location': '東京・両国国技館',
 249             'duration': 18044,
 250             'timestamp': 1647665400,
 251             'upload_date': '20220319',
 252             'thumbnail': 'https://image.asset.wrestle-universe.com/i8jxSTCHPfdAKD4zN41Psx/i8jxSTCHPfdAKD4zN41Psx',
 253             'thumbnails': 'count:3',
 254         },
 255         'params': {
 256             'skip_download': 'm3u8',
 257         },
 258     }]
 259
 260     _API_PATH = 'events'
 261
 262     def _real_extract(self, url):
 263         lang, video_id = self._match_valid_url(url).group('lang', 'id')
 264         metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
 265
 266         info = {
 267             'id': video_id,
 268             **traverse_obj(metadata, {
 269                 'title': ('displayName', {str}),
 270                 'description': ('description', {str}),
 271                 'channel': ('labels', 'group', {str}),
 272                 'location': ('labels', 'venue', {str}),
 273                 'timestamp': ('startTime', {int_or_none}),
 274                 'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
 275             }),
 276         }
 277
 278         ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
 279         if info.get('timestamp') and ended_time:
 280             info['duration'] = ended_time - info['timestamp']
 281
 282         video_data, decrypt = self._call_encrypted_api(
 283             video_id, ':watchArchive', 'watch archive', data={'method': 1})
 284         info['formats'] = self._get_formats(video_data, (
 285             ('hls', None), ('urls', 'chromecastUrls'), ..., {url_or_none}), video_id)
 286         for f in info['formats']:
 287             # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
 288             if f.get('tbr'):
 289                 f['tbr'] = int(f['tbr'] / 2.5)
 290
 291         hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
 292         if hls_aes_key:
 293             info['hls_aes'] = {
 294                 'key': hls_aes_key,
 295                 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
 296             },
 297         elif traverse_obj(video_data, ('hls', 'encryptType', {int})):
 298             self.report_warning('HLS AES-128 key was not found in API response')
 299
 300         return info