]>
Commit | Line | Data |
---|---|---|
1 | import base64 | |
2 | import binascii | |
3 | import json | |
4 | import time | |
5 | import uuid | |
6 | ||
7 | from .common import InfoExtractor | |
8 | from ..dependencies import Cryptodome | |
9 | from ..utils import ( | |
10 | ExtractorError, | |
11 | int_or_none, | |
12 | jwt_decode_hs256, | |
13 | traverse_obj, | |
14 | try_call, | |
15 | url_basename, | |
16 | url_or_none, | |
17 | urlencode_postdata, | |
18 | variadic, | |
19 | ) | |
20 | ||
21 | ||
22 | class WrestleUniverseBaseIE(InfoExtractor): | |
23 | _NETRC_MACHINE = 'wrestleuniverse' | |
24 | _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)' | |
25 | _API_HOST = 'api.wrestle-universe.com' | |
26 | _API_PATH = None | |
27 | _REAL_TOKEN = None | |
28 | _TOKEN_EXPIRY = None | |
29 | _REFRESH_TOKEN = None | |
30 | _DEVICE_ID = None | |
31 | _LOGIN_QUERY = {'key': 'AIzaSyCaRPBsDQYVDUWWBXjsTrHESi2r_F3RAdA'} | |
32 | _LOGIN_HEADERS = { | |
33 | 'Accept': '*/*', | |
34 | 'Content-Type': 'application/json', | |
35 | 'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web', | |
36 | 'X-Firebase-gmpid': '1:307308870738:web:820f38fe5150c8976e338b', | |
37 | 'Referer': 'https://www.wrestle-universe.com/', | |
38 | 'Origin': 'https://www.wrestle-universe.com', | |
39 | } | |
40 | ||
41 | @property | |
42 | def _TOKEN(self): | |
43 | if not self._REAL_TOKEN or not self._TOKEN_EXPIRY: | |
44 | token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value) | |
45 | if not token and not self._REFRESH_TOKEN: | |
46 | self.raise_login_required() | |
47 | self._TOKEN = token | |
48 | ||
49 | if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()): | |
50 | if not self._REFRESH_TOKEN: | |
51 | raise ExtractorError( | |
52 | 'Expired token. Refresh your cookies in browser and try again', expected=True) | |
53 | self._refresh_token() | |
54 | ||
55 | return self._REAL_TOKEN | |
56 | ||
57 | @_TOKEN.setter | |
58 | def _TOKEN(self, value): | |
59 | self._REAL_TOKEN = value | |
60 | ||
61 | expiry = traverse_obj(value, ({jwt_decode_hs256}, 'exp', {int_or_none})) | |
62 | if not expiry: | |
63 | raise ExtractorError('There was a problem with the auth token') | |
64 | self._TOKEN_EXPIRY = expiry | |
65 | ||
66 | def _perform_login(self, username, password): | |
67 | login = self._download_json( | |
68 | 'https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword', None, | |
69 | 'Logging in', query=self._LOGIN_QUERY, headers=self._LOGIN_HEADERS, data=json.dumps({ | |
70 | 'returnSecureToken': True, | |
71 | 'email': username, | |
72 | 'password': password, | |
73 | }, separators=(',', ':')).encode(), expected_status=400) | |
74 | token = traverse_obj(login, ('idToken', {str})) | |
75 | if not token: | |
76 | raise ExtractorError( | |
77 | f'Unable to log in: {traverse_obj(login, ("error", "message"))}', expected=True) | |
78 | self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str})) | |
79 | if not self._REFRESH_TOKEN: | |
80 | self.report_warning('No refresh token was granted') | |
81 | self._TOKEN = token | |
82 | ||
83 | def _real_initialize(self): | |
84 | if self._DEVICE_ID: | |
85 | return | |
86 | ||
87 | self._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key=self._NETRC_MACHINE)[0] | |
88 | if not self._DEVICE_ID: | |
89 | self._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id') | |
90 | if self._DEVICE_ID: | |
91 | return | |
92 | self._DEVICE_ID = str(uuid.uuid4()) | |
93 | ||
94 | self.cache.store(self._NETRC_MACHINE, 'device_id', self._DEVICE_ID) | |
95 | ||
96 | def _refresh_token(self): | |
97 | refresh = self._download_json( | |
98 | 'https://securetoken.googleapis.com/v1/token', None, 'Refreshing token', | |
99 | query=self._LOGIN_QUERY, data=urlencode_postdata({ | |
100 | 'grant_type': 'refresh_token', | |
101 | 'refresh_token': self._REFRESH_TOKEN, | |
102 | }), headers={ | |
103 | **self._LOGIN_HEADERS, | |
104 | 'Content-Type': 'application/x-www-form-urlencoded', | |
105 | }) | |
106 | if traverse_obj(refresh, ('refresh_token', {str})): | |
107 | self._REFRESH_TOKEN = refresh['refresh_token'] | |
108 | token = traverse_obj(refresh, 'access_token', 'id_token', expected_type=str) | |
109 | if not token: | |
110 | raise ExtractorError('No auth token returned from refresh request') | |
111 | self._TOKEN = token | |
112 | ||
113 | def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True): | |
114 | headers = {'CA-CID': ''} | |
115 | if data: | |
116 | headers['Content-Type'] = 'application/json;charset=utf-8' | |
117 | data = json.dumps(data, separators=(',', ':')).encode() | |
118 | if auth and self._TOKEN: | |
119 | headers['Authorization'] = f'Bearer {self._TOKEN}' | |
120 | return self._download_json( | |
121 | f'https://{self._API_HOST}/v1/{self._API_PATH}/{video_id}{param}', video_id, | |
122 | note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON', | |
123 | data=data, headers=headers, query=query, fatal=fatal) | |
124 | ||
125 | def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True): | |
126 | if not Cryptodome.RSA: | |
127 | raise ExtractorError('pycryptodomex not found. Please install', expected=True) | |
128 | private_key = Cryptodome.RSA.generate(2048) | |
129 | cipher = Cryptodome.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.SHA1) | |
130 | ||
131 | def decrypt(data): | |
132 | if not data: | |
133 | return None | |
134 | try: | |
135 | return cipher.decrypt(base64.b64decode(data)).decode() | |
136 | except (ValueError, binascii.Error) as e: | |
137 | raise ExtractorError(f'Could not decrypt data: {e}') | |
138 | ||
139 | token = base64.b64encode(private_key.public_key().export_key('DER')).decode() | |
140 | api_json = self._call_api(video_id, param, msg, data={ | |
141 | 'deviceId': self._DEVICE_ID, | |
142 | 'token': token, | |
143 | **data, | |
144 | }, query=query, fatal=fatal) | |
145 | return api_json, decrypt | |
146 | ||
147 | def _download_metadata(self, url, video_id, lang, props_keys): | |
148 | metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False) | |
149 | if not metadata: | |
150 | webpage = self._download_webpage(url, video_id) | |
151 | nextjs_data = self._search_nextjs_data(webpage, video_id, fatal=False) | |
152 | metadata = traverse_obj(nextjs_data, ( | |
153 | 'props', 'pageProps', *variadic(props_keys, (str, bytes, dict, set)), {dict})) or {} | |
154 | return metadata | |
155 | ||
156 | def _get_formats(self, data, path, video_id=None): | |
157 | hls_url = traverse_obj(data, path, get_all=False) | |
158 | if not hls_url and not data.get('canWatch'): | |
159 | self.raise_no_formats( | |
160 | 'This account does not have access to the requested content', expected=True) | |
161 | elif not hls_url: | |
162 | self.raise_no_formats('No supported formats found') | |
163 | return self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', live=True) | |
164 | ||
165 | ||
166 | class WrestleUniverseVODIE(WrestleUniverseBaseIE): | |
167 | _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'videos' | |
168 | _TESTS = [{ | |
169 | 'url': 'https://www.wrestle-universe.com/en/videos/dp8mpjmcKfxzUhEHM2uFws', | |
170 | 'info_dict': { | |
171 | 'id': 'dp8mpjmcKfxzUhEHM2uFws', | |
172 | 'ext': 'mp4', | |
173 | 'title': 'The 3rd “Futari wa Princess” Max Heart Tournament', | |
174 | 'description': 'md5:318d5061e944797fbbb81d5c7dd00bf5', | |
175 | 'location': '埼玉・春日部ふれあいキューブ', | |
176 | 'channel': 'tjpw', | |
177 | 'duration': 7119, | |
178 | 'timestamp': 1674979200, | |
179 | 'upload_date': '20230129', | |
180 | 'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN', | |
181 | 'chapters': 'count:7', | |
182 | 'cast': 'count:21', | |
183 | }, | |
184 | 'params': { | |
185 | 'skip_download': 'm3u8', | |
186 | }, | |
187 | }] | |
188 | ||
189 | _API_PATH = 'videoEpisodes' | |
190 | ||
191 | def _real_extract(self, url): | |
192 | lang, video_id = self._match_valid_url(url).group('lang', 'id') | |
193 | metadata = self._download_metadata(url, video_id, lang, 'videoEpisodeFallbackData') | |
194 | video_data = self._call_api(video_id, ':watch', 'watch', data={'deviceId': self._DEVICE_ID}) | |
195 | ||
196 | return { | |
197 | 'id': video_id, | |
198 | 'formats': self._get_formats(video_data, ('protocolHls', 'url', {url_or_none}), video_id), | |
199 | **traverse_obj(metadata, { | |
200 | 'title': ('displayName', {str}), | |
201 | 'description': ('description', {str}), | |
202 | 'channel': ('labels', 'group', {str}), | |
203 | 'location': ('labels', 'venue', {str}), | |
204 | 'timestamp': ('watchStartTime', {int_or_none}), | |
205 | 'thumbnail': ('keyVisualUrl', {url_or_none}), | |
206 | 'cast': ('casts', ..., 'displayName', {str}), | |
207 | 'duration': ('duration', {int}), | |
208 | 'chapters': ('videoChapters', lambda _, v: isinstance(v.get('start'), int), { | |
209 | 'title': ('displayName', {str}), | |
210 | 'start_time': ('start', {int}), | |
211 | 'end_time': ('end', {int}), | |
212 | }), | |
213 | }), | |
214 | } | |
215 | ||
216 | ||
217 | class WrestleUniversePPVIE(WrestleUniverseBaseIE): | |
218 | _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'lives' | |
219 | _TESTS = [{ | |
220 | 'note': 'HLS AES-128 key obtained via API', | |
221 | 'url': 'https://www.wrestle-universe.com/en/lives/buH9ibbfhdJAY4GKZcEuJX', | |
222 | 'info_dict': { | |
223 | 'id': 'buH9ibbfhdJAY4GKZcEuJX', | |
224 | 'ext': 'mp4', | |
225 | 'title': '【PPV】Beyond the origins, into the future', | |
226 | 'description': 'md5:9a872db68cd09be4a1e35a3ee8b0bdfc', | |
227 | 'channel': 'tjpw', | |
228 | 'location': '東京・Twin Box AKIHABARA', | |
229 | 'duration': 10098, | |
230 | 'timestamp': 1675076400, | |
231 | 'upload_date': '20230130', | |
232 | 'thumbnail': 'https://image.asset.wrestle-universe.com/rJs2m7cBaLXrwCcxMdQGRM/rJs2m7cBaLXrwCcxMdQGRM', | |
233 | 'thumbnails': 'count:3', | |
234 | 'hls_aes': { | |
235 | 'key': '5633184acd6e43f1f1ac71c6447a4186', | |
236 | 'iv': '5bac71beb33197d5600337ce86de7862', | |
237 | }, | |
238 | }, | |
239 | 'params': { | |
240 | 'skip_download': 'm3u8', | |
241 | }, | |
242 | 'skip': 'No longer available', | |
243 | }, { | |
244 | 'note': 'unencrypted HLS', | |
245 | 'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx', | |
246 | 'info_dict': { | |
247 | 'id': 'wUG8hP5iApC63jbtQzhVVx', | |
248 | 'ext': 'mp4', | |
249 | 'title': 'GRAND PRINCESS \'22', | |
250 | 'description': 'md5:e4f43d0d4262de3952ff34831bc99858', | |
251 | 'channel': 'tjpw', | |
252 | 'location': '東京・両国国技館', | |
253 | 'duration': 18044, | |
254 | 'timestamp': 1647665400, | |
255 | 'upload_date': '20220319', | |
256 | 'thumbnail': 'https://image.asset.wrestle-universe.com/i8jxSTCHPfdAKD4zN41Psx/i8jxSTCHPfdAKD4zN41Psx', | |
257 | 'thumbnails': 'count:3', | |
258 | }, | |
259 | 'params': { | |
260 | 'skip_download': 'm3u8', | |
261 | }, | |
262 | }, { | |
263 | 'note': 'manifest provides live-a (partial) and live-b (full) streams', | |
264 | 'url': 'https://www.wrestle-universe.com/en/lives/umc99R9XsexXrxr9VjTo9g', | |
265 | 'only_matching': True, | |
266 | }] | |
267 | ||
268 | _API_PATH = 'events' | |
269 | ||
270 | def _real_extract(self, url): | |
271 | lang, video_id = self._match_valid_url(url).group('lang', 'id') | |
272 | metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData') | |
273 | ||
274 | info = { | |
275 | 'id': video_id, | |
276 | **traverse_obj(metadata, { | |
277 | 'title': ('displayName', {str}), | |
278 | 'description': ('description', {str}), | |
279 | 'channel': ('labels', 'group', {str}), | |
280 | 'location': ('labels', 'venue', {str}), | |
281 | 'timestamp': ('startTime', {int_or_none}), | |
282 | 'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}), | |
283 | }), | |
284 | } | |
285 | ||
286 | ended_time = traverse_obj(metadata, ('endedTime', {int_or_none})) | |
287 | if info.get('timestamp') and ended_time: | |
288 | info['duration'] = ended_time - info['timestamp'] | |
289 | ||
290 | video_data, decrypt = self._call_encrypted_api( | |
291 | video_id, ':watchArchive', 'watch archive', data={'method': 1}) | |
292 | # 'chromecastUrls' can be only partial videos, avoid | |
293 | info['formats'] = self._get_formats(video_data, ('hls', (('urls', ...), 'url'), {url_or_none}), video_id) | |
294 | for f in info['formats']: | |
295 | # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values | |
296 | if f.get('tbr'): | |
297 | f['tbr'] = int(f['tbr'] / 2.5) | |
298 | # prefer variants with the same basename as the master playlist to avoid partial streams | |
299 | f['format_id'] = url_basename(f['url']).partition('.')[0] | |
300 | if not f['format_id'].startswith(url_basename(f['manifest_url']).partition('.')[0]): | |
301 | f['preference'] = -10 | |
302 | ||
303 | hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt})) | |
304 | if hls_aes_key: | |
305 | info['hls_aes'] = { | |
306 | 'key': hls_aes_key, | |
307 | 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})), | |
308 | } | |
309 | elif traverse_obj(video_data, ('hls', 'encryptType', {int})): | |
310 | self.report_warning('HLS AES-128 key was not found in API response') | |
311 | ||
312 | return info |