]>
Commit | Line | Data |
---|---|---|
fada8272 JJ |
1 | import base64 |
2 | import binascii | |
3 | import datetime | |
4 | import hashlib | |
5 | import hmac | |
6 | import json | |
7 | import os | |
8 | ||
9 | from .common import InfoExtractor | |
10 | from ..utils import ( | |
11 | ExtractorError, | |
12 | traverse_obj, | |
13 | unescapeHTML, | |
14 | ) | |
15 | ||
16 | ||
17 | class GoPlayIE(InfoExtractor): | |
18 | _VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/]+/[^/]+/|)(?P<display_id>[^/#]+)' | |
19 | ||
20 | _NETRC_MACHINE = 'goplay' | |
21 | ||
22 | _TESTS = [{ | |
23 | 'url': 'https://www.goplay.be/video/de-container-cup/de-container-cup-s3/de-container-cup-s3-aflevering-2#autoplay', | |
24 | 'info_dict': { | |
25 | 'id': '9c4214b8-e55d-4e4b-a446-f015f6c6f811', | |
26 | 'ext': 'mp4', | |
27 | 'title': 'S3 - Aflevering 2', | |
28 | 'series': 'De Container Cup', | |
29 | 'season': 'Season 3', | |
30 | 'season_number': 3, | |
31 | 'episode': 'Episode 2', | |
32 | 'episode_number': 2, | |
33 | }, | |
34 | 'skip': 'This video is only available for registered users' | |
35 | }, { | |
36 | 'url': 'https://www.goplay.be/video/a-family-for-thr-holidays-s1-aflevering-1#autoplay', | |
37 | 'info_dict': { | |
38 | 'id': '74e3ed07-748c-49e4-85a0-393a93337dbf', | |
39 | 'ext': 'mp4', | |
40 | 'title': 'A Family for the Holidays', | |
41 | }, | |
42 | 'skip': 'This video is only available for registered users' | |
7e90e34f | 43 | }, { |
44 | 'url': 'https://www.goplay.be/video/de-mol/de-mol-s11/de-mol-s11-aflevering-1#autoplay', | |
45 | 'info_dict': { | |
46 | 'id': '03eb8f2f-153e-41cb-9805-0d3a29dab656', | |
47 | 'ext': 'mp4', | |
48 | 'title': 'S11 - Aflevering 1', | |
49 | 'episode': 'Episode 1', | |
50 | 'series': 'De Mol', | |
51 | 'season_number': 11, | |
52 | 'episode_number': 1, | |
53 | 'season': 'Season 11' | |
54 | }, | |
55 | 'params': { | |
56 | 'skip_download': True | |
57 | }, | |
58 | 'skip': 'This video is only available for registered users' | |
fada8272 JJ |
59 | }] |
60 | ||
61 | _id_token = None | |
62 | ||
63 | def _perform_login(self, username, password): | |
64 | self.report_login() | |
65 | aws = AwsIdp(ie=self, pool_id='eu-west-1_dViSsKM5Y', client_id='6s1h851s8uplco5h6mqh1jac8m') | |
66 | self._id_token, _ = aws.authenticate(username=username, password=password) | |
67 | ||
68 | def _real_initialize(self): | |
69 | if not self._id_token: | |
70 | raise self.raise_login_required(method='password') | |
71 | ||
72 | def _real_extract(self, url): | |
73 | url, display_id = self._match_valid_url(url).group(0, 'display_id') | |
74 | webpage = self._download_webpage(url, display_id) | |
75 | video_data_json = self._html_search_regex(r'<div\s+data-hero="([^"]+)"', webpage, 'video_data') | |
76 | video_data = self._parse_json(unescapeHTML(video_data_json), display_id).get('data') | |
77 | ||
78 | movie = video_data.get('movie') | |
79 | if movie: | |
80 | video_id = movie['videoUuid'] | |
81 | info_dict = { | |
82 | 'title': movie.get('title') | |
83 | } | |
84 | else: | |
85 | episode = traverse_obj(video_data, ('playlists', ..., 'episodes', lambda _, v: v['pageInfo']['url'] == url), get_all=False) | |
86 | video_id = episode['videoUuid'] | |
87 | info_dict = { | |
88 | 'title': episode.get('episodeTitle'), | |
89 | 'series': traverse_obj(episode, ('program', 'title')), | |
90 | 'season_number': episode.get('seasonNumber'), | |
91 | 'episode_number': episode.get('episodeNumber'), | |
92 | } | |
93 | ||
94 | api = self._download_json( | |
d27bde98 | 95 | f'https://api.goplay.be/web/v1/videos/long-form/{video_id}', |
7e90e34f | 96 | video_id, headers={ |
97 | 'Authorization': 'Bearer %s' % self._id_token, | |
98 | **self.geo_verification_headers(), | |
99 | }) | |
100 | ||
101 | if 'manifestUrls' in api: | |
102 | formats, subtitles = self._extract_m3u8_formats_and_subtitles( | |
103 | api['manifestUrls']['hls'], video_id, ext='mp4', m3u8_id='HLS') | |
fada8272 | 104 | |
7e90e34f | 105 | else: |
106 | if 'ssai' not in api: | |
107 | raise ExtractorError('expecting Google SSAI stream') | |
108 | ||
109 | ssai_content_source_id = api['ssai']['contentSourceID'] | |
110 | ssai_video_id = api['ssai']['videoID'] | |
111 | ||
112 | dai = self._download_json( | |
113 | f'https://dai.google.com/ondemand/dash/content/{ssai_content_source_id}/vid/{ssai_video_id}/streams', | |
114 | video_id, data=b'{"api-key":"null"}', | |
115 | headers={'content-type': 'application/json'}) | |
116 | ||
117 | periods = self._extract_mpd_periods(dai['stream_manifest'], video_id) | |
118 | ||
119 | # skip pre-roll and mid-roll ads | |
120 | periods = [p for p in periods if '-ad-' not in p['id']] | |
121 | ||
122 | formats, subtitles = self._merge_mpd_periods(periods) | |
fada8272 JJ |
123 | |
124 | info_dict.update({ | |
125 | 'id': video_id, | |
126 | 'formats': formats, | |
7e90e34f | 127 | 'subtitles': subtitles, |
fada8272 | 128 | }) |
fada8272 JJ |
129 | return info_dict |
130 | ||
131 | ||
132 | # Taken from https://github.com/add-ons/plugin.video.viervijfzes/blob/master/resources/lib/viervijfzes/auth_awsidp.py | |
133 | # Released into Public domain by https://github.com/michaelarnauts | |
134 | ||
135 | class InvalidLoginException(ExtractorError): | |
136 | """ The login credentials are invalid """ | |
137 | ||
138 | ||
139 | class AuthenticationException(ExtractorError): | |
140 | """ Something went wrong while logging in """ | |
141 | ||
142 | ||
143 | class AwsIdp: | |
144 | """ AWS Identity Provider """ | |
145 | ||
146 | def __init__(self, ie, pool_id, client_id): | |
147 | """ | |
148 | :param InfoExtrator ie: The extractor that instantiated this class. | |
149 | :param str pool_id: The AWS user pool to connect to (format: <region>_<poolid>). | |
150 | E.g.: eu-west-1_aLkOfYN3T | |
151 | :param str client_id: The client application ID (the ID of the application connecting) | |
152 | """ | |
153 | ||
154 | self.ie = ie | |
155 | ||
156 | self.pool_id = pool_id | |
157 | if "_" not in self.pool_id: | |
158 | raise ValueError("Invalid pool_id format. Should be <region>_<poolid>.") | |
159 | ||
160 | self.client_id = client_id | |
161 | self.region = self.pool_id.split("_")[0] | |
162 | self.url = "https://cognito-idp.%s.amazonaws.com/" % (self.region,) | |
163 | ||
164 | # Initialize the values | |
165 | # https://github.com/aws/amazon-cognito-identity-js/blob/master/src/AuthenticationHelper.js#L22 | |
166 | self.n_hex = 'FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1' + \ | |
167 | '29024E088A67CC74020BBEA63B139B22514A08798E3404DD' + \ | |
168 | 'EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245' + \ | |
169 | 'E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED' + \ | |
170 | 'EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D' + \ | |
171 | 'C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F' + \ | |
172 | '83655D23DCA3AD961C62F356208552BB9ED529077096966D' + \ | |
173 | '670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B' + \ | |
174 | 'E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9' + \ | |
175 | 'DE2BCBF6955817183995497CEA956AE515D2261898FA0510' + \ | |
176 | '15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64' + \ | |
177 | 'ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7' + \ | |
178 | 'ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B' + \ | |
179 | 'F12FFA06D98A0864D87602733EC86A64521F2B18177B200C' + \ | |
180 | 'BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31' + \ | |
181 | '43DB5BFCE0FD108E4B82D120A93AD2CAFFFFFFFFFFFFFFFF' | |
182 | ||
183 | # https://github.com/aws/amazon-cognito-identity-js/blob/master/src/AuthenticationHelper.js#L49 | |
184 | self.g_hex = '2' | |
185 | self.info_bits = bytearray('Caldera Derived Key', 'utf-8') | |
186 | ||
187 | self.big_n = self.__hex_to_long(self.n_hex) | |
188 | self.g = self.__hex_to_long(self.g_hex) | |
189 | self.k = self.__hex_to_long(self.__hex_hash('00' + self.n_hex + '0' + self.g_hex)) | |
190 | self.small_a_value = self.__generate_random_small_a() | |
191 | self.large_a_value = self.__calculate_a() | |
192 | ||
193 | def authenticate(self, username, password): | |
194 | """ Authenticate with a username and password. """ | |
195 | # Step 1: First initiate an authentication request | |
196 | auth_data_dict = self.__get_authentication_request(username) | |
197 | auth_data = json.dumps(auth_data_dict).encode("utf-8") | |
198 | auth_headers = { | |
199 | "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth", | |
200 | "Accept-Encoding": "identity", | |
201 | "Content-Type": "application/x-amz-json-1.1" | |
202 | } | |
203 | auth_response_json = self.ie._download_json( | |
204 | self.url, None, data=auth_data, headers=auth_headers, | |
205 | note='Authenticating username', errnote='Invalid username') | |
206 | challenge_parameters = auth_response_json.get("ChallengeParameters") | |
207 | ||
208 | if auth_response_json.get("ChallengeName") != "PASSWORD_VERIFIER": | |
209 | raise AuthenticationException(auth_response_json["message"]) | |
210 | ||
211 | # Step 2: Respond to the Challenge with a valid ChallengeResponse | |
212 | challenge_request = self.__get_challenge_response_request(challenge_parameters, password) | |
213 | challenge_data = json.dumps(challenge_request).encode("utf-8") | |
214 | challenge_headers = { | |
215 | "X-Amz-Target": "AWSCognitoIdentityProviderService.RespondToAuthChallenge", | |
216 | "Content-Type": "application/x-amz-json-1.1" | |
217 | } | |
218 | auth_response_json = self.ie._download_json( | |
219 | self.url, None, data=challenge_data, headers=challenge_headers, | |
220 | note='Authenticating password', errnote='Invalid password') | |
221 | ||
222 | if 'message' in auth_response_json: | |
223 | raise InvalidLoginException(auth_response_json['message']) | |
224 | return ( | |
225 | auth_response_json['AuthenticationResult']['IdToken'], | |
226 | auth_response_json['AuthenticationResult']['RefreshToken'] | |
227 | ) | |
228 | ||
229 | def __get_authentication_request(self, username): | |
230 | """ | |
231 | ||
232 | :param str username: The username to use | |
233 | ||
234 | :return: A full Authorization request. | |
235 | :rtype: dict | |
236 | """ | |
237 | auth_request = { | |
238 | "AuthParameters": { | |
239 | "USERNAME": username, | |
240 | "SRP_A": self.__long_to_hex(self.large_a_value) | |
241 | }, | |
242 | "AuthFlow": "USER_SRP_AUTH", | |
243 | "ClientId": self.client_id | |
244 | } | |
245 | return auth_request | |
246 | ||
247 | def __get_challenge_response_request(self, challenge_parameters, password): | |
248 | """ Create a Challenge Response Request object. | |
249 | ||
250 | :param dict[str,str|imt] challenge_parameters: The parameters for the challenge. | |
251 | :param str password: The password. | |
252 | ||
253 | :return: A valid and full request data object to use as a response for a challenge. | |
254 | :rtype: dict | |
255 | """ | |
256 | user_id = challenge_parameters["USERNAME"] | |
257 | user_id_for_srp = challenge_parameters["USER_ID_FOR_SRP"] | |
258 | srp_b = challenge_parameters["SRP_B"] | |
259 | salt = challenge_parameters["SALT"] | |
260 | secret_block = challenge_parameters["SECRET_BLOCK"] | |
261 | ||
262 | timestamp = self.__get_current_timestamp() | |
263 | ||
264 | # Get a HKDF key for the password, SrpB and the Salt | |
265 | hkdf = self.__get_hkdf_key_for_password( | |
266 | user_id_for_srp, | |
267 | password, | |
268 | self.__hex_to_long(srp_b), | |
269 | salt | |
270 | ) | |
271 | secret_block_bytes = base64.standard_b64decode(secret_block) | |
272 | ||
273 | # the message is a combo of the pool_id, provided SRP userId, the Secret and Timestamp | |
274 | msg = \ | |
275 | bytearray(self.pool_id.split('_')[1], 'utf-8') + \ | |
276 | bytearray(user_id_for_srp, 'utf-8') + \ | |
277 | bytearray(secret_block_bytes) + \ | |
278 | bytearray(timestamp, 'utf-8') | |
279 | hmac_obj = hmac.new(hkdf, msg, digestmod=hashlib.sha256) | |
280 | signature_string = base64.standard_b64encode(hmac_obj.digest()).decode('utf-8') | |
281 | challenge_request = { | |
282 | "ChallengeResponses": { | |
283 | "USERNAME": user_id, | |
284 | "TIMESTAMP": timestamp, | |
285 | "PASSWORD_CLAIM_SECRET_BLOCK": secret_block, | |
286 | "PASSWORD_CLAIM_SIGNATURE": signature_string | |
287 | }, | |
288 | "ChallengeName": "PASSWORD_VERIFIER", | |
289 | "ClientId": self.client_id | |
290 | } | |
291 | return challenge_request | |
292 | ||
293 | def __get_hkdf_key_for_password(self, username, password, server_b_value, salt): | |
294 | """ Calculates the final hkdf based on computed S value, and computed U value and the key. | |
295 | ||
296 | :param str username: Username. | |
297 | :param str password: Password. | |
298 | :param int server_b_value: Server B value. | |
299 | :param int salt: Generated salt. | |
300 | ||
301 | :return Computed HKDF value. | |
302 | :rtype: object | |
303 | """ | |
304 | ||
305 | u_value = self.__calculate_u(self.large_a_value, server_b_value) | |
306 | if u_value == 0: | |
307 | raise ValueError('U cannot be zero.') | |
308 | username_password = '%s%s:%s' % (self.pool_id.split('_')[1], username, password) | |
309 | username_password_hash = self.__hash_sha256(username_password.encode('utf-8')) | |
310 | ||
311 | x_value = self.__hex_to_long(self.__hex_hash(self.__pad_hex(salt) + username_password_hash)) | |
312 | g_mod_pow_xn = pow(self.g, x_value, self.big_n) | |
313 | int_value2 = server_b_value - self.k * g_mod_pow_xn | |
314 | s_value = pow(int_value2, self.small_a_value + u_value * x_value, self.big_n) | |
315 | hkdf = self.__compute_hkdf( | |
316 | bytearray.fromhex(self.__pad_hex(s_value)), | |
317 | bytearray.fromhex(self.__pad_hex(self.__long_to_hex(u_value))) | |
318 | ) | |
319 | return hkdf | |
320 | ||
321 | def __compute_hkdf(self, ikm, salt): | |
322 | """ Standard hkdf algorithm | |
323 | ||
324 | :param {Buffer} ikm Input key material. | |
325 | :param {Buffer} salt Salt value. | |
326 | :return {Buffer} Strong key material. | |
327 | """ | |
328 | ||
329 | prk = hmac.new(salt, ikm, hashlib.sha256).digest() | |
330 | info_bits_update = self.info_bits + bytearray(chr(1), 'utf-8') | |
331 | hmac_hash = hmac.new(prk, info_bits_update, hashlib.sha256).digest() | |
332 | return hmac_hash[:16] | |
333 | ||
334 | def __calculate_u(self, big_a, big_b): | |
335 | """ Calculate the client's value U which is the hash of A and B | |
336 | ||
337 | :param int big_a: Large A value. | |
338 | :param int big_b: Server B value. | |
339 | ||
340 | :return Computed U value. | |
341 | :rtype: int | |
342 | """ | |
343 | ||
344 | u_hex_hash = self.__hex_hash(self.__pad_hex(big_a) + self.__pad_hex(big_b)) | |
345 | return self.__hex_to_long(u_hex_hash) | |
346 | ||
347 | def __generate_random_small_a(self): | |
348 | """ Helper function to generate a random big integer | |
349 | ||
350 | :return a random value. | |
351 | :rtype: int | |
352 | """ | |
353 | random_long_int = self.__get_random(128) | |
354 | return random_long_int % self.big_n | |
355 | ||
356 | def __calculate_a(self): | |
357 | """ Calculate the client's public value A = g^a%N with the generated random number a | |
358 | ||
359 | :return Computed large A. | |
360 | :rtype: int | |
361 | """ | |
362 | ||
363 | big_a = pow(self.g, self.small_a_value, self.big_n) | |
364 | # safety check | |
365 | if (big_a % self.big_n) == 0: | |
366 | raise ValueError('Safety check for A failed') | |
367 | return big_a | |
368 | ||
369 | @staticmethod | |
370 | def __long_to_hex(long_num): | |
371 | return '%x' % long_num | |
372 | ||
373 | @staticmethod | |
374 | def __hex_to_long(hex_string): | |
375 | return int(hex_string, 16) | |
376 | ||
377 | @staticmethod | |
378 | def __hex_hash(hex_string): | |
379 | return AwsIdp.__hash_sha256(bytearray.fromhex(hex_string)) | |
380 | ||
381 | @staticmethod | |
382 | def __hash_sha256(buf): | |
383 | """AuthenticationHelper.hash""" | |
384 | digest = hashlib.sha256(buf).hexdigest() | |
385 | return (64 - len(digest)) * '0' + digest | |
386 | ||
387 | @staticmethod | |
388 | def __pad_hex(long_int): | |
389 | """ Converts a Long integer (or hex string) to hex format padded with zeroes for hashing | |
390 | ||
391 | :param int|str long_int: Number or string to pad. | |
392 | ||
393 | :return Padded hex string. | |
394 | :rtype: str | |
395 | """ | |
396 | ||
397 | if not isinstance(long_int, str): | |
398 | hash_str = AwsIdp.__long_to_hex(long_int) | |
399 | else: | |
400 | hash_str = long_int | |
401 | if len(hash_str) % 2 == 1: | |
402 | hash_str = '0%s' % hash_str | |
403 | elif hash_str[0] in '89ABCDEFabcdef': | |
404 | hash_str = '00%s' % hash_str | |
405 | return hash_str | |
406 | ||
407 | @staticmethod | |
408 | def __get_random(nbytes): | |
409 | random_hex = binascii.hexlify(os.urandom(nbytes)) | |
410 | return AwsIdp.__hex_to_long(random_hex) | |
411 | ||
412 | @staticmethod | |
413 | def __get_current_timestamp(): | |
414 | """ Creates a timestamp with the correct English format. | |
415 | ||
416 | :return: timestamp in format 'Sun Jan 27 19:00:04 UTC 2019' | |
417 | :rtype: str | |
418 | """ | |
419 | ||
420 | # We need US only data, so we cannot just do a strftime: | |
421 | # Sun Jan 27 19:00:04 UTC 2019 | |
422 | months = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] | |
423 | days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] | |
424 | ||
836e06d2 | 425 | time_now = datetime.datetime.now(datetime.timezone.utc) |
fada8272 | 426 | format_string = "{} {} {} %H:%M:%S UTC %Y".format(days[time_now.weekday()], months[time_now.month], time_now.day) |
836e06d2 | 427 | time_string = time_now.strftime(format_string) |
fada8272 JJ |
428 | return time_string |
429 | ||
430 | def __str__(self): | |
431 | return "AWS IDP Client for:\nRegion: %s\nPoolId: %s\nAppId: %s" % ( | |
432 | self.region, self.pool_id.split("_")[1], self.client_id | |
433 | ) |