]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/wrestleuniverse.py
[extractor/stacommu] Add extractors (#7432)
[yt-dlp.git] / yt_dlp / extractor / wrestleuniverse.py
CommitLineData
e61acb40 1import base64
2import binascii
3import json
4import time
ef8fb7f0 5import uuid
e61acb40 6
7from .common import InfoExtractor
8from ..dependencies import Cryptodome
9from ..utils import (
10 ExtractorError,
11 int_or_none,
12 jwt_decode_hs256,
13 traverse_obj,
14 try_call,
15 url_or_none,
ef8fb7f0 16 urlencode_postdata,
af1fd12f 17 variadic,
e61acb40 18)
19
20
21class WrestleUniverseBaseIE(InfoExtractor):
ef8fb7f0 22 _NETRC_MACHINE = 'wrestleuniverse'
e61acb40 23 _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
af1fd12f 24 _API_HOST = 'api.wrestle-universe.com'
e61acb40 25 _API_PATH = None
ef8fb7f0 26 _REAL_TOKEN = None
e61acb40 27 _TOKEN_EXPIRY = None
ef8fb7f0 28 _REFRESH_TOKEN = None
29 _DEVICE_ID = None
30 _LOGIN_QUERY = {'key': 'AIzaSyCaRPBsDQYVDUWWBXjsTrHESi2r_F3RAdA'}
31 _LOGIN_HEADERS = {
32 'Accept': '*/*',
33 'Content-Type': 'application/json',
34 'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
35 'X-Firebase-gmpid': '1:307308870738:web:820f38fe5150c8976e338b',
36 'Referer': 'https://www.wrestle-universe.com/',
37 'Origin': 'https://www.wrestle-universe.com',
38 }
e61acb40 39
ef8fb7f0 40 @property
41 def _TOKEN(self):
42 if not self._REAL_TOKEN or not self._TOKEN_EXPIRY:
43 token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
44 if not token and not self._REFRESH_TOKEN:
e61acb40 45 self.raise_login_required()
c8561c6d 46 self._TOKEN = token
e61acb40 47
ef8fb7f0 48 if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
49 if not self._REFRESH_TOKEN:
50 raise ExtractorError(
51 'Expired token. Refresh your cookies in browser and try again', expected=True)
52 self._refresh_token()
e61acb40 53
ef8fb7f0 54 return self._REAL_TOKEN
55
56 @_TOKEN.setter
57 def _TOKEN(self, value):
58 self._REAL_TOKEN = value
59
60 expiry = traverse_obj(value, ({jwt_decode_hs256}, 'exp', {int_or_none}))
61 if not expiry:
62 raise ExtractorError('There was a problem with the auth token')
63 self._TOKEN_EXPIRY = expiry
64
65 def _perform_login(self, username, password):
66 login = self._download_json(
67 'https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword', None,
68 'Logging in', query=self._LOGIN_QUERY, headers=self._LOGIN_HEADERS, data=json.dumps({
69 'returnSecureToken': True,
70 'email': username,
71 'password': password,
af1fd12f 72 }, separators=(',', ':')).encode(), expected_status=400)
73 token = traverse_obj(login, ('idToken', {str}))
74 if not token:
75 raise ExtractorError(
76 f'Unable to log in: {traverse_obj(login, ("error", "message"))}', expected=True)
ef8fb7f0 77 self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
78 if not self._REFRESH_TOKEN:
79 self.report_warning('No refresh token was granted')
af1fd12f 80 self._TOKEN = token
ef8fb7f0 81
82 def _real_initialize(self):
af1fd12f 83 if self._DEVICE_ID:
ef8fb7f0 84 return
85
af1fd12f 86 self._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key=self._NETRC_MACHINE)[0]
87 if not self._DEVICE_ID:
88 self._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
89 if self._DEVICE_ID:
ef8fb7f0 90 return
af1fd12f 91 self._DEVICE_ID = str(uuid.uuid4())
ef8fb7f0 92
af1fd12f 93 self.cache.store(self._NETRC_MACHINE, 'device_id', self._DEVICE_ID)
ef8fb7f0 94
95 def _refresh_token(self):
96 refresh = self._download_json(
97 'https://securetoken.googleapis.com/v1/token', None, 'Refreshing token',
98 query=self._LOGIN_QUERY, data=urlencode_postdata({
99 'grant_type': 'refresh_token',
100 'refresh_token': self._REFRESH_TOKEN,
101 }), headers={
102 **self._LOGIN_HEADERS,
103 'Content-Type': 'application/x-www-form-urlencoded',
104 })
105 if traverse_obj(refresh, ('refresh_token', {str})):
106 self._REFRESH_TOKEN = refresh['refresh_token']
107 token = traverse_obj(refresh, 'access_token', 'id_token', expected_type=str)
108 if not token:
109 raise ExtractorError('No auth token returned from refresh request')
110 self._TOKEN = token
e61acb40 111
112 def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
113 headers = {'CA-CID': ''}
114 if data:
115 headers['Content-Type'] = 'application/json;charset=utf-8'
116 data = json.dumps(data, separators=(',', ':')).encode()
af1fd12f 117 if auth and self._TOKEN:
ef8fb7f0 118 headers['Authorization'] = f'Bearer {self._TOKEN}'
e61acb40 119 return self._download_json(
af1fd12f 120 f'https://{self._API_HOST}/v1/{self._API_PATH}/{video_id}{param}', video_id,
e61acb40 121 note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
122 data=data, headers=headers, query=query, fatal=fatal)
123
124 def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True):
65f6e807 125 if not Cryptodome.RSA:
e61acb40 126 raise ExtractorError('pycryptodomex not found. Please install', expected=True)
65f6e807 127 private_key = Cryptodome.RSA.generate(2048)
128 cipher = Cryptodome.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.SHA1)
e61acb40 129
130 def decrypt(data):
131 if not data:
132 return None
133 try:
134 return cipher.decrypt(base64.b64decode(data)).decode()
135 except (ValueError, binascii.Error) as e:
136 raise ExtractorError(f'Could not decrypt data: {e}')
137
138 token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
139 api_json = self._call_api(video_id, param, msg, data={
ef8fb7f0 140 'deviceId': self._DEVICE_ID,
e61acb40 141 'token': token,
142 **data,
143 }, query=query, fatal=fatal)
144 return api_json, decrypt
145
af1fd12f 146 def _download_metadata(self, url, video_id, lang, props_keys):
e61acb40 147 metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
148 if not metadata:
149 webpage = self._download_webpage(url, video_id)
150 nextjs_data = self._search_nextjs_data(webpage, video_id)
af1fd12f 151 metadata = traverse_obj(nextjs_data, (
152 'props', 'pageProps', *variadic(props_keys, (str, bytes, dict, set)), {dict})) or {}
e61acb40 153 return metadata
154
155 def _get_formats(self, data, path, video_id=None):
156 hls_url = traverse_obj(data, path, get_all=False)
157 if not hls_url and not data.get('canWatch'):
158 self.raise_no_formats(
159 'This account does not have access to the requested content', expected=True)
160 elif not hls_url:
161 self.raise_no_formats('No supported formats found')
162 return self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', live=True)
163
164
165class WrestleUniverseVODIE(WrestleUniverseBaseIE):
166 _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'videos'
167 _TESTS = [{
168 'url': 'https://www.wrestle-universe.com/en/videos/dp8mpjmcKfxzUhEHM2uFws',
169 'info_dict': {
170 'id': 'dp8mpjmcKfxzUhEHM2uFws',
171 'ext': 'mp4',
172 'title': 'The 3rd “Futari wa Princess” Max Heart Tournament',
173 'description': 'md5:318d5061e944797fbbb81d5c7dd00bf5',
174 'location': '埼玉・春日部ふれあいキューブ',
175 'channel': 'tjpw',
176 'duration': 7119,
177 'timestamp': 1674979200,
178 'upload_date': '20230129',
179 'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
180 'chapters': 'count:7',
ef8fb7f0 181 'cast': 'count:21',
e61acb40 182 },
183 'params': {
184 'skip_download': 'm3u8',
185 },
186 }]
187
188 _API_PATH = 'videoEpisodes'
189
190 def _real_extract(self, url):
191 lang, video_id = self._match_valid_url(url).group('lang', 'id')
192 metadata = self._download_metadata(url, video_id, lang, 'videoEpisodeFallbackData')
193 video_data = self._call_api(video_id, ':watch', 'watch', data={
194 # 'deviceId' is required if ignoreDeviceRestriction is False
195 'ignoreDeviceRestriction': True,
196 })
197
198 return {
199 'id': video_id,
200 'formats': self._get_formats(video_data, (
201 (('protocolHls', 'url'), ('chromecastUrls', ...)), {url_or_none}), video_id),
202 **traverse_obj(metadata, {
203 'title': ('displayName', {str}),
204 'description': ('description', {str}),
205 'channel': ('labels', 'group', {str}),
206 'location': ('labels', 'venue', {str}),
207 'timestamp': ('watchStartTime', {int_or_none}),
208 'thumbnail': ('keyVisualUrl', {url_or_none}),
209 'cast': ('casts', ..., 'displayName', {str}),
210 'duration': ('duration', {int}),
211 'chapters': ('videoChapters', lambda _, v: isinstance(v.get('start'), int), {
212 'title': ('displayName', {str}),
213 'start_time': ('start', {int}),
214 'end_time': ('end', {int}),
215 }),
216 }),
217 }
218
219
220class WrestleUniversePPVIE(WrestleUniverseBaseIE):
221 _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'lives'
222 _TESTS = [{
223 'note': 'HLS AES-128 key obtained via API',
224 'url': 'https://www.wrestle-universe.com/en/lives/buH9ibbfhdJAY4GKZcEuJX',
225 'info_dict': {
226 'id': 'buH9ibbfhdJAY4GKZcEuJX',
227 'ext': 'mp4',
228 'title': '【PPV】Beyond the origins, into the future',
229 'description': 'md5:9a872db68cd09be4a1e35a3ee8b0bdfc',
230 'channel': 'tjpw',
231 'location': '東京・Twin Box AKIHABARA',
232 'duration': 10098,
233 'timestamp': 1675076400,
234 'upload_date': '20230130',
235 'thumbnail': 'https://image.asset.wrestle-universe.com/rJs2m7cBaLXrwCcxMdQGRM/rJs2m7cBaLXrwCcxMdQGRM',
236 'thumbnails': 'count:3',
237 'hls_aes': {
238 'key': '5633184acd6e43f1f1ac71c6447a4186',
239 'iv': '5bac71beb33197d5600337ce86de7862',
240 },
241 },
242 'params': {
243 'skip_download': 'm3u8',
244 },
ef8fb7f0 245 'skip': 'No longer available',
e61acb40 246 }, {
247 'note': 'unencrypted HLS',
248 'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
249 'info_dict': {
250 'id': 'wUG8hP5iApC63jbtQzhVVx',
251 'ext': 'mp4',
252 'title': 'GRAND PRINCESS \'22',
253 'description': 'md5:e4f43d0d4262de3952ff34831bc99858',
254 'channel': 'tjpw',
255 'location': '東京・両国国技館',
256 'duration': 18044,
257 'timestamp': 1647665400,
258 'upload_date': '20220319',
259 'thumbnail': 'https://image.asset.wrestle-universe.com/i8jxSTCHPfdAKD4zN41Psx/i8jxSTCHPfdAKD4zN41Psx',
260 'thumbnails': 'count:3',
261 },
262 'params': {
263 'skip_download': 'm3u8',
264 },
265 }]
266
267 _API_PATH = 'events'
268
269 def _real_extract(self, url):
270 lang, video_id = self._match_valid_url(url).group('lang', 'id')
271 metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
272
ef8fb7f0 273 info = {
274 'id': video_id,
275 **traverse_obj(metadata, {
276 'title': ('displayName', {str}),
277 'description': ('description', {str}),
278 'channel': ('labels', 'group', {str}),
279 'location': ('labels', 'venue', {str}),
280 'timestamp': ('startTime', {int_or_none}),
281 'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
282 }),
283 }
e61acb40 284
285 ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
286 if info.get('timestamp') and ended_time:
287 info['duration'] = ended_time - info['timestamp']
288
289 video_data, decrypt = self._call_encrypted_api(
290 video_id, ':watchArchive', 'watch archive', data={'method': 1})
ef8fb7f0 291 info['formats'] = self._get_formats(video_data, (
e61acb40 292 ('hls', None), ('urls', 'chromecastUrls'), ..., {url_or_none}), video_id)
ef8fb7f0 293 for f in info['formats']:
e61acb40 294 # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
295 if f.get('tbr'):
296 f['tbr'] = int(f['tbr'] / 2.5)
297
298 hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
ef8fb7f0 299 if hls_aes_key:
300 info['hls_aes'] = {
e61acb40 301 'key': hls_aes_key,
302 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
303 },
ef8fb7f0 304 elif traverse_obj(video_data, ('hls', 'encryptType', {int})):
305 self.report_warning('HLS AES-128 key was not found in API response')
306
307 return info