]>
Commit | Line | Data |
---|---|---|
706dfe44 | 1 | import base64 |
954e57e4 | 2 | import uuid |
ac668111 | 3 | |
46279958 | 4 | from .common import InfoExtractor |
f2816634 | 5 | from ..networking import Request |
3d2623a8 | 6 | from ..networking.exceptions import HTTPError |
1cc79574 PH |
7 | from ..utils import ( |
8 | ExtractorError, | |
54a5be4d | 9 | float_or_none, |
706dfe44 | 10 | format_field, |
032de83e | 11 | int_or_none, |
954e57e4 | 12 | jwt_decode_hs256, |
032de83e SS |
13 | parse_age_limit, |
14 | parse_count, | |
b99ba3df | 15 | parse_iso8601, |
a9d4da60 | 16 | qualities, |
032de83e | 17 | time_seconds, |
706dfe44 | 18 | traverse_obj, |
032de83e SS |
19 | url_or_none, |
20 | urlencode_postdata, | |
c8434e83 | 21 | ) |
c8434e83 | 22 | |
34440095 | 23 | |
46279958 | 24 | class CrunchyrollBaseIE(InfoExtractor): |
032de83e | 25 | _BASE_URL = 'https://www.crunchyroll.com' |
7c74a015 | 26 | _API_BASE = 'https://api.crunchyroll.com' |
80f48920 | 27 | _NETRC_MACHINE = 'crunchyroll' |
f2816634 | 28 | _SWITCH_USER_AGENT = 'Crunchyroll/1.8.0 Nintendo Switch/12.3.12.0 UE4/4.27' |
ff38a011 | 29 | _REFRESH_TOKEN = None |
032de83e | 30 | _AUTH_HEADERS = None |
ff38a011 | 31 | _AUTH_EXPIRY = None |
032de83e | 32 | _API_ENDPOINT = None |
ff38a011 | 33 | _BASIC_AUTH = 'Basic ' + base64.b64encode(':'.join(( |
34 | 't-kdgp2h8c3jub8fn0fq', | |
35 | 'yfLDfMfrYvKXh4JXS1LEI2cCqu1v5Wan', | |
36 | )).encode()).decode() | |
954e57e4 | 37 | _IS_PREMIUM = None |
9b16762f SS |
38 | _LOCALE_LOOKUP = { |
39 | 'ar': 'ar-SA', | |
40 | 'de': 'de-DE', | |
41 | '': 'en-US', | |
42 | 'es': 'es-419', | |
43 | 'es-es': 'es-ES', | |
44 | 'fr': 'fr-FR', | |
45 | 'it': 'it-IT', | |
46 | 'pt-br': 'pt-BR', | |
47 | 'pt-pt': 'pt-PT', | |
48 | 'ru': 'ru-RU', | |
49 | 'hi': 'hi-IN', | |
50 | } | |
05dee6c5 | 51 | |
ff38a011 | 52 | def _set_auth_info(self, response): |
53 | CrunchyrollBaseIE._IS_PREMIUM = 'cr_premium' in traverse_obj(response, ('access_token', {jwt_decode_hs256}, 'benefits', ...)) | |
54 | CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': response['token_type'] + ' ' + response['access_token']} | |
55 | CrunchyrollBaseIE._AUTH_EXPIRY = time_seconds(seconds=traverse_obj(response, ('expires_in', {float_or_none}), default=300) - 10) | |
56 | ||
57 | def _request_token(self, headers, data, note='Requesting token', errnote='Failed to request token'): | |
5904853a | 58 | try: |
ff38a011 | 59 | return self._download_json( |
60 | f'{self._BASE_URL}/auth/v1/token', None, note=note, errnote=errnote, | |
5904853a | 61 | headers=headers, data=urlencode_postdata(data), impersonate=True) |
ff38a011 | 62 | except ExtractorError as error: |
63 | if not isinstance(error.cause, HTTPError) or error.cause.status != 403: | |
64 | raise | |
5904853a | 65 | if target := error.cause.response.extensions.get('impersonate'): |
66 | raise ExtractorError(f'Got HTTP Error 403 when using impersonate target "{target}"') | |
ff38a011 | 67 | raise ExtractorError( |
5904853a | 68 | 'Request blocked by Cloudflare. ' |
69 | 'Install the required impersonation dependency if possible, ' | |
70 | 'or else navigate to Crunchyroll in your browser, ' | |
ff38a011 | 71 | 'then pass the fresh cookies (with --cookies-from-browser or --cookies) ' |
72 | 'and your browser\'s User-Agent (with --user-agent)', expected=True) | |
44699d10 | 73 | |
52efa4b3 | 74 | def _perform_login(self, username, password): |
ff38a011 | 75 | if not CrunchyrollBaseIE._REFRESH_TOKEN: |
76 | CrunchyrollBaseIE._REFRESH_TOKEN = self.cache.load(self._NETRC_MACHINE, username) | |
77 | if CrunchyrollBaseIE._REFRESH_TOKEN: | |
eb5b1fc0 S |
78 | return |
79 | ||
ff38a011 | 80 | try: |
81 | login_response = self._request_token( | |
82 | headers={'Authorization': self._BASIC_AUTH}, data={ | |
83 | 'username': username, | |
84 | 'password': password, | |
85 | 'grant_type': 'password', | |
86 | 'scope': 'offline_access', | |
87 | }, note='Logging in', errnote='Failed to log in') | |
88 | except ExtractorError as error: | |
89 | if isinstance(error.cause, HTTPError) and error.cause.status == 401: | |
90 | raise ExtractorError('Invalid username and/or password', expected=True) | |
91 | raise | |
92 | ||
93 | CrunchyrollBaseIE._REFRESH_TOKEN = login_response['refresh_token'] | |
94 | self.cache.store(self._NETRC_MACHINE, username, CrunchyrollBaseIE._REFRESH_TOKEN) | |
95 | self._set_auth_info(login_response) | |
80f48920 | 96 | |
032de83e | 97 | def _update_auth(self): |
ff38a011 | 98 | if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_EXPIRY > time_seconds(): |
032de83e SS |
99 | return |
100 | ||
ff38a011 | 101 | auth_headers = {'Authorization': self._BASIC_AUTH} |
102 | if CrunchyrollBaseIE._REFRESH_TOKEN: | |
103 | data = { | |
104 | 'refresh_token': CrunchyrollBaseIE._REFRESH_TOKEN, | |
105 | 'grant_type': 'refresh_token', | |
106 | 'scope': 'offline_access', | |
107 | } | |
954e57e4 | 108 | else: |
ff38a011 | 109 | data = {'grant_type': 'client_id'} |
954e57e4 | 110 | auth_headers['ETP-Anonymous-ID'] = uuid.uuid4() |
9b16762f | 111 | try: |
ff38a011 | 112 | auth_response = self._request_token(auth_headers, data) |
9b16762f | 113 | except ExtractorError as error: |
ff38a011 | 114 | username, password = self._get_login_info() |
115 | if not username or not isinstance(error.cause, HTTPError) or error.cause.status != 400: | |
116 | raise | |
117 | self.to_screen('Refresh token has expired. Re-logging in') | |
118 | CrunchyrollBaseIE._REFRESH_TOKEN = None | |
119 | self.cache.store(self._NETRC_MACHINE, username, None) | |
120 | self._perform_login(username, password) | |
121 | return | |
032de83e | 122 | |
ff38a011 | 123 | self._set_auth_info(auth_response) |
032de83e | 124 | |
9b16762f SS |
125 | def _locale_from_language(self, language): |
126 | config_locale = self._configuration_arg('metadata', ie_key=CrunchyrollBetaIE, casesense=True) | |
127 | return config_locale[0] if config_locale else self._LOCALE_LOOKUP.get(language) | |
128 | ||
032de83e | 129 | def _call_base_api(self, endpoint, internal_id, lang, note=None, query={}): |
032de83e SS |
130 | self._update_auth() |
131 | ||
132 | if not endpoint.startswith('/'): | |
133 | endpoint = f'/{endpoint}' | |
134 | ||
9b16762f SS |
135 | query = query.copy() |
136 | locale = self._locale_from_language(lang) | |
137 | if locale: | |
138 | query['locale'] = locale | |
139 | ||
032de83e SS |
140 | return self._download_json( |
141 | f'{self._BASE_URL}{endpoint}', internal_id, note or f'Calling API: {endpoint}', | |
9b16762f | 142 | headers=CrunchyrollBaseIE._AUTH_HEADERS, query=query) |
032de83e SS |
143 | |
144 | def _call_api(self, path, internal_id, lang, note='api', query={}): | |
145 | if not path.startswith(f'/content/v2/{self._API_ENDPOINT}/'): | |
146 | path = f'/content/v2/{self._API_ENDPOINT}/{path}' | |
147 | ||
148 | try: | |
149 | result = self._call_base_api( | |
150 | path, internal_id, lang, f'Downloading {note} JSON ({self._API_ENDPOINT})', query=query) | |
151 | except ExtractorError as error: | |
3d2623a8 | 152 | if isinstance(error.cause, HTTPError) and error.cause.status == 404: |
032de83e SS |
153 | return None |
154 | raise | |
155 | ||
156 | if not result: | |
157 | raise ExtractorError(f'Unexpected response when downloading {note} JSON') | |
158 | return result | |
159 | ||
954e57e4 | 160 | def _extract_chapters(self, internal_id): |
161 | # if no skip events are available, a 403 xml error is returned | |
162 | skip_events = self._download_json( | |
163 | f'https://static.crunchyroll.com/skip-events/production/{internal_id}.json', | |
164 | internal_id, note='Downloading chapter info', fatal=False, errnote=False) | |
165 | if not skip_events: | |
166 | return None | |
167 | ||
168 | chapters = [] | |
169 | for event in ('recap', 'intro', 'credits', 'preview'): | |
170 | start = traverse_obj(skip_events, (event, 'start', {float_or_none})) | |
171 | end = traverse_obj(skip_events, (event, 'end', {float_or_none})) | |
172 | # some chapters have no start and/or ending time, they will just be ignored | |
173 | if start is None or end is None: | |
032de83e | 174 | continue |
954e57e4 | 175 | chapters.append({'title': event.capitalize(), 'start_time': start, 'end_time': end}) |
176 | ||
177 | return chapters | |
178 | ||
179 | def _extract_stream(self, identifier, display_id=None): | |
180 | if not display_id: | |
181 | display_id = identifier | |
182 | ||
183 | self._update_auth() | |
f2816634 | 184 | headers = {**CrunchyrollBaseIE._AUTH_HEADERS, 'User-Agent': self._SWITCH_USER_AGENT} |
185 | try: | |
186 | stream_response = self._download_json( | |
187 | f'https://cr-play-service.prd.crunchyrollsvc.com/v1/{identifier}/console/switch/play', | |
188 | display_id, note='Downloading stream info', errnote='Failed to download stream info', headers=headers) | |
189 | except ExtractorError as error: | |
190 | if self.get_param('ignore_no_formats_error'): | |
191 | self.report_warning(error.orig_msg) | |
192 | return [], {} | |
193 | elif isinstance(error.cause, HTTPError) and error.cause.status == 420: | |
194 | raise ExtractorError( | |
195 | 'You have reached the rate-limit for active streams; try again later', expected=True) | |
196 | raise | |
954e57e4 | 197 | |
198 | available_formats = {'': ('', '', stream_response['url'])} | |
199 | for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])): | |
200 | available_formats[hardsub_lang] = (f'hardsub-{hardsub_lang}', hardsub_lang, stream['url']) | |
032de83e SS |
201 | |
202 | requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])] | |
954e57e4 | 203 | hardsub_langs = [lang for lang in available_formats if lang] |
204 | if hardsub_langs and 'all' not in requested_hardsubs: | |
032de83e | 205 | full_format_langs = set(requested_hardsubs) |
954e57e4 | 206 | self.to_screen(f'Available hardsub languages: {", ".join(hardsub_langs)}') |
032de83e | 207 | self.to_screen( |
954e57e4 | 208 | 'To extract formats of a hardsub language, use ' |
032de83e SS |
209 | '"--extractor-args crunchyrollbeta:hardsub=<language_code or all>". ' |
210 | 'See https://github.com/yt-dlp/yt-dlp#crunchyrollbeta-crunchyroll for more info', | |
211 | only_once=True) | |
212 | else: | |
213 | full_format_langs = set(map(str.lower, available_formats)) | |
214 | ||
954e57e4 | 215 | audio_locale = traverse_obj(stream_response, ('audioLocale', {str})) |
032de83e | 216 | hardsub_preference = qualities(requested_hardsubs[::-1]) |
954e57e4 | 217 | formats, subtitles = [], {} |
218 | for format_id, hardsub_lang, stream_url in available_formats.values(): | |
219 | if hardsub_lang.lower() in full_format_langs: | |
220 | adaptive_formats, dash_subs = self._extract_mpd_formats_and_subtitles( | |
221 | stream_url, display_id, mpd_id=format_id, headers=CrunchyrollBaseIE._AUTH_HEADERS, | |
222 | fatal=False, note=f'Downloading {f"{format_id} " if hardsub_lang else ""}MPD manifest') | |
223 | self._merge_subtitles(dash_subs, target=subtitles) | |
459262ac | 224 | else: |
f2816634 | 225 | continue # XXX: Update this if meta mpd formats work; will be tricky with token invalidation |
032de83e SS |
226 | for f in adaptive_formats: |
227 | if f.get('acodec') != 'none': | |
228 | f['language'] = audio_locale | |
229 | f['quality'] = hardsub_preference(hardsub_lang.lower()) | |
230 | formats.extend(adaptive_formats) | |
231 | ||
954e57e4 | 232 | for locale, subtitle in traverse_obj(stream_response, (('subtitles', 'captions'), {dict.items}, ...)): |
233 | subtitles.setdefault(locale, []).append(traverse_obj(subtitle, {'url': 'url', 'ext': 'format'})) | |
032de83e | 234 | |
f2816634 | 235 | # Invalidate stream token to avoid rate-limit |
236 | error_msg = 'Unable to invalidate stream token; you may experience rate-limiting' | |
237 | if stream_token := stream_response.get('token'): | |
238 | self._request_webpage(Request( | |
239 | f'https://cr-play-service.prd.crunchyrollsvc.com/v1/token/{identifier}/{stream_token}/inactive', | |
240 | headers=headers, method='PATCH'), display_id, 'Invalidating stream token', error_msg, fatal=False) | |
241 | else: | |
242 | self.report_warning(error_msg) | |
243 | ||
954e57e4 | 244 | return formats, subtitles |
032de83e SS |
245 | |
246 | ||
247 | class CrunchyrollCmsBaseIE(CrunchyrollBaseIE): | |
248 | _API_ENDPOINT = 'cms' | |
249 | _CMS_EXPIRY = None | |
250 | ||
251 | def _call_cms_api_signed(self, path, internal_id, lang, note='api'): | |
252 | if not CrunchyrollCmsBaseIE._CMS_EXPIRY or CrunchyrollCmsBaseIE._CMS_EXPIRY <= time_seconds(): | |
253 | response = self._call_base_api('index/v2', None, lang, 'Retrieving signed policy')['cms_web'] | |
254 | CrunchyrollCmsBaseIE._CMS_QUERY = { | |
255 | 'Policy': response['policy'], | |
256 | 'Signature': response['signature'], | |
257 | 'Key-Pair-Id': response['key_pair_id'], | |
f4d706a9 | 258 | } |
032de83e SS |
259 | CrunchyrollCmsBaseIE._CMS_BUCKET = response['bucket'] |
260 | CrunchyrollCmsBaseIE._CMS_EXPIRY = parse_iso8601(response['expires']) - 10 | |
261 | ||
262 | if not path.startswith('/cms/v2'): | |
263 | path = f'/cms/v2{CrunchyrollCmsBaseIE._CMS_BUCKET}/{path}' | |
f4d706a9 | 264 | |
032de83e SS |
265 | return self._call_base_api( |
266 | path, internal_id, lang, f'Downloading {note} JSON (signed cms)', query=CrunchyrollCmsBaseIE._CMS_QUERY) | |
f4d706a9 | 267 | |
032de83e SS |
268 | |
269 | class CrunchyrollBetaIE(CrunchyrollCmsBaseIE): | |
cb1553e9 | 270 | IE_NAME = 'crunchyroll' |
5da42f2b | 271 | _VALID_URL = r'''(?x) |
032de83e | 272 | https?://(?:beta\.|www\.)?crunchyroll\.com/ |
9b16762f | 273 | (?:(?P<lang>\w{2}(?:-\w{2})?)/)? |
032de83e | 274 | watch/(?!concert|musicvideo)(?P<id>\w+)''' |
dd078970 | 275 | _TESTS = [{ |
032de83e | 276 | # Premium only |
cb1553e9 | 277 | 'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future', |
dd078970 | 278 | 'info_dict': { |
b99ba3df | 279 | 'id': 'GY2P1Q98Y', |
dd078970 | 280 | 'ext': 'mp4', |
b99ba3df JH |
281 | 'duration': 1380.241, |
282 | 'timestamp': 1459632600, | |
dd078970 | 283 | 'description': 'md5:a022fbec4fbb023d43631032c91ed64b', |
dd078970 | 284 | 'title': 'World Trigger Episode 73 – To the Future', |
285 | 'upload_date': '20160402', | |
f4d706a9 | 286 | 'series': 'World Trigger', |
b99ba3df | 287 | 'series_id': 'GR757DMKY', |
f4d706a9 | 288 | 'season': 'World Trigger', |
b99ba3df | 289 | 'season_id': 'GR9P39NJ6', |
f4d706a9 | 290 | 'season_number': 1, |
b99ba3df JH |
291 | 'episode': 'To the Future', |
292 | 'episode_number': 73, | |
032de83e | 293 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', |
93abb740 | 294 | 'chapters': 'count:2', |
032de83e SS |
295 | 'age_limit': 14, |
296 | 'like_count': int, | |
297 | 'dislike_count': int, | |
dd078970 | 298 | }, |
954e57e4 | 299 | 'params': { |
300 | 'skip_download': 'm3u8', | |
301 | 'extractor_args': {'crunchyrollbeta': {'hardsub': ['de-DE']}}, | |
302 | 'format': 'bv[format_id~=hardsub]', | |
303 | }, | |
dfea94f8 | 304 | }, { |
032de83e | 305 | # Premium only |
cb1553e9 | 306 | 'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR', |
dfea94f8 SS |
307 | 'info_dict': { |
308 | 'id': 'GYE5WKQGR', | |
309 | 'ext': 'mp4', | |
310 | 'duration': 366.459, | |
311 | 'timestamp': 1476788400, | |
312 | 'description': 'md5:74b67283ffddd75f6e224ca7dc031e76', | |
032de83e | 313 | 'title': 'SHELTER – Porter Robinson presents Shelter the Animation', |
dfea94f8 SS |
314 | 'upload_date': '20161018', |
315 | 'series': 'SHELTER', | |
316 | 'series_id': 'GYGG09WWY', | |
317 | 'season': 'SHELTER', | |
318 | 'season_id': 'GR09MGK4R', | |
319 | 'season_number': 1, | |
320 | 'episode': 'Porter Robinson presents Shelter the Animation', | |
321 | 'episode_number': 0, | |
032de83e SS |
322 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', |
323 | 'age_limit': 14, | |
324 | 'like_count': int, | |
325 | 'dislike_count': int, | |
dfea94f8 SS |
326 | }, |
327 | 'params': {'skip_download': True}, | |
032de83e SS |
328 | }, { |
329 | 'url': 'https://www.crunchyroll.com/watch/GJWU2VKK3/cherry-blossom-meeting-and-a-coming-blizzard', | |
330 | 'info_dict': { | |
331 | 'id': 'GJWU2VKK3', | |
332 | 'ext': 'mp4', | |
333 | 'duration': 1420.054, | |
334 | 'description': 'md5:2d1c67c0ec6ae514d9c30b0b99a625cd', | |
335 | 'title': 'The Ice Guy and His Cool Female Colleague Episode 1 – Cherry Blossom Meeting and a Coming Blizzard', | |
336 | 'series': 'The Ice Guy and His Cool Female Colleague', | |
337 | 'series_id': 'GW4HM75NP', | |
338 | 'season': 'The Ice Guy and His Cool Female Colleague', | |
339 | 'season_id': 'GY9PC21VE', | |
340 | 'season_number': 1, | |
341 | 'episode': 'Cherry Blossom Meeting and a Coming Blizzard', | |
342 | 'episode_number': 1, | |
343 | 'chapters': 'count:2', | |
344 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', | |
345 | 'timestamp': 1672839000, | |
346 | 'upload_date': '20230104', | |
347 | 'age_limit': 14, | |
348 | 'like_count': int, | |
349 | 'dislike_count': int, | |
350 | }, | |
351 | 'params': {'skip_download': 'm3u8'}, | |
352 | }, { | |
353 | 'url': 'https://www.crunchyroll.com/watch/GM8F313NQ', | |
354 | 'info_dict': { | |
355 | 'id': 'GM8F313NQ', | |
356 | 'ext': 'mp4', | |
357 | 'title': 'Garakowa -Restore the World-', | |
358 | 'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608', | |
359 | 'duration': 3996.104, | |
360 | 'age_limit': 13, | |
361 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', | |
362 | }, | |
363 | 'params': {'skip_download': 'm3u8'}, | |
954e57e4 | 364 | 'skip': 'no longer exists', |
032de83e SS |
365 | }, { |
366 | 'url': 'https://www.crunchyroll.com/watch/G62PEZ2E6', | |
367 | 'info_dict': { | |
368 | 'id': 'G62PEZ2E6', | |
369 | 'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608', | |
370 | 'age_limit': 13, | |
371 | 'duration': 65.138, | |
372 | 'title': 'Garakowa -Restore the World-', | |
373 | }, | |
374 | 'playlist_mincount': 5, | |
f4d706a9 | 375 | }, { |
9b16762f | 376 | 'url': 'https://www.crunchyroll.com/de/watch/GY2P1Q98Y', |
f4d706a9 | 377 | 'only_matching': True, |
964b5493 | 378 | }, { |
379 | 'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy', | |
380 | 'only_matching': True, | |
dd078970 | 381 | }] |
032de83e SS |
382 | # We want to support lazy playlist filtering and movie listings cannot be inside a playlist |
383 | _RETURN_TYPE = 'video' | |
dd078970 | 384 | |
385 | def _real_extract(self, url): | |
032de83e | 386 | lang, internal_id = self._match_valid_url(url).group('lang', 'id') |
f4d706a9 | 387 | |
032de83e SS |
388 | # We need to use unsigned API call to allow ratings query string |
389 | response = traverse_obj(self._call_api( | |
390 | f'objects/{internal_id}', internal_id, lang, 'object info', {'ratings': 'true'}), ('data', 0, {dict})) | |
391 | if not response: | |
392 | raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True) | |
706dfe44 | 393 | |
032de83e SS |
394 | object_type = response.get('type') |
395 | if object_type == 'episode': | |
396 | result = self._transform_episode_response(response) | |
706dfe44 | 397 | |
032de83e SS |
398 | elif object_type == 'movie': |
399 | result = self._transform_movie_response(response) | |
706dfe44 | 400 | |
032de83e SS |
401 | elif object_type == 'movie_listing': |
402 | first_movie_id = traverse_obj(response, ('movie_listing_metadata', 'first_movie_id')) | |
403 | if not self._yes_playlist(internal_id, first_movie_id): | |
404 | return self.url_result(f'{self._BASE_URL}/{lang}watch/{first_movie_id}', CrunchyrollBetaIE, first_movie_id) | |
405 | ||
406 | def entries(): | |
407 | movies = self._call_api(f'movie_listings/{internal_id}/movies', internal_id, lang, 'movie list') | |
408 | for movie_response in traverse_obj(movies, ('data', ...)): | |
409 | yield self.url_result( | |
410 | f'{self._BASE_URL}/{lang}watch/{movie_response["id"]}', | |
411 | CrunchyrollBetaIE, **self._transform_movie_response(movie_response)) | |
412 | ||
413 | return self.playlist_result(entries(), **self._transform_movie_response(response)) | |
dfea94f8 | 414 | |
dfea94f8 | 415 | else: |
032de83e | 416 | raise ExtractorError(f'Unknown object type {object_type}') |
dfea94f8 | 417 | |
954e57e4 | 418 | if not self._IS_PREMIUM and traverse_obj(response, (f'{object_type}_metadata', 'is_premium_only')): |
032de83e | 419 | message = f'This {object_type} is for premium members only' |
ff38a011 | 420 | if CrunchyrollBaseIE._REFRESH_TOKEN: |
cb2fb4a6 | 421 | self.raise_no_formats(message, expected=True, video_id=internal_id) |
422 | else: | |
423 | self.raise_login_required(message, method='password', metadata_available=True) | |
424 | else: | |
425 | result['formats'], result['subtitles'] = self._extract_stream(internal_id) | |
706dfe44 | 426 | |
954e57e4 | 427 | result['chapters'] = self._extract_chapters(internal_id) |
93abb740 | 428 | |
032de83e SS |
429 | def calculate_count(item): |
430 | return parse_count(''.join((item['displayed'], item.get('unit') or ''))) | |
431 | ||
432 | result.update(traverse_obj(response, ('rating', { | |
433 | 'like_count': ('up', {calculate_count}), | |
434 | 'dislike_count': ('down', {calculate_count}), | |
435 | }))) | |
436 | ||
437 | return result | |
438 | ||
439 | @staticmethod | |
440 | def _transform_episode_response(data): | |
441 | metadata = traverse_obj(data, (('episode_metadata', None), {dict}), get_all=False) or {} | |
706dfe44 | 442 | return { |
032de83e SS |
443 | 'id': data['id'], |
444 | 'title': ' \u2013 '.join(( | |
add96eb9 | 445 | ('{}{}'.format( |
032de83e SS |
446 | format_field(metadata, 'season_title'), |
447 | format_field(metadata, 'episode', ' Episode %s'))), | |
448 | format_field(data, 'title'))), | |
449 | **traverse_obj(data, { | |
450 | 'episode': ('title', {str}), | |
451 | 'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}), | |
452 | 'thumbnails': ('images', 'thumbnail', ..., ..., { | |
453 | 'url': ('source', {url_or_none}), | |
454 | 'width': ('width', {int_or_none}), | |
455 | 'height': ('height', {int_or_none}), | |
456 | }), | |
457 | }), | |
458 | **traverse_obj(metadata, { | |
459 | 'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}), | |
460 | 'timestamp': ('upload_date', {parse_iso8601}), | |
461 | 'series': ('series_title', {str}), | |
462 | 'series_id': ('series_id', {str}), | |
463 | 'season': ('season_title', {str}), | |
464 | 'season_id': ('season_id', {str}), | |
465 | 'season_number': ('season_number', ({int}, {float_or_none})), | |
466 | 'episode_number': ('sequence_number', ({int}, {float_or_none})), | |
467 | 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), | |
468 | 'language': ('audio_locale', {str}), | |
469 | }, get_all=False), | |
706dfe44 | 470 | } |
dd078970 | 471 | |
032de83e SS |
472 | @staticmethod |
473 | def _transform_movie_response(data): | |
474 | metadata = traverse_obj(data, (('movie_metadata', 'movie_listing_metadata', None), {dict}), get_all=False) or {} | |
475 | return { | |
476 | 'id': data['id'], | |
477 | **traverse_obj(data, { | |
478 | 'title': ('title', {str}), | |
479 | 'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}), | |
480 | 'thumbnails': ('images', 'thumbnail', ..., ..., { | |
481 | 'url': ('source', {url_or_none}), | |
482 | 'width': ('width', {int_or_none}), | |
483 | 'height': ('height', {int_or_none}), | |
484 | }), | |
485 | }), | |
486 | **traverse_obj(metadata, { | |
487 | 'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}), | |
488 | 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), | |
489 | }), | |
490 | } | |
dd078970 | 491 | |
032de83e SS |
492 | |
493 | class CrunchyrollBetaShowIE(CrunchyrollCmsBaseIE): | |
cb1553e9 | 494 | IE_NAME = 'crunchyroll:playlist' |
5da42f2b | 495 | _VALID_URL = r'''(?x) |
032de83e | 496 | https?://(?:beta\.|www\.)?crunchyroll\.com/ |
5da42f2b | 497 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) |
032de83e | 498 | series/(?P<id>\w+)''' |
dd078970 | 499 | _TESTS = [{ |
cb1553e9 | 500 | 'url': 'https://www.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA', |
dd078970 | 501 | 'info_dict': { |
b99ba3df | 502 | 'id': 'GY19NQ2QR', |
dd078970 | 503 | 'title': 'Girl Friend BETA', |
032de83e SS |
504 | 'description': 'md5:99c1b22ee30a74b536a8277ced8eb750', |
505 | # XXX: `thumbnail` does not get set from `thumbnails` in playlist | |
506 | # 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', | |
507 | 'age_limit': 14, | |
dd078970 | 508 | }, |
509 | 'playlist_mincount': 10, | |
510 | }, { | |
5da42f2b | 511 | 'url': 'https://beta.crunchyroll.com/it/series/GY19NQ2QR', |
dd078970 | 512 | 'only_matching': True, |
513 | }] | |
514 | ||
515 | def _real_extract(self, url): | |
032de83e SS |
516 | lang, internal_id = self._match_valid_url(url).group('lang', 'id') |
517 | ||
518 | def entries(): | |
519 | seasons_response = self._call_cms_api_signed(f'seasons?series_id={internal_id}', internal_id, lang, 'seasons') | |
520 | for season in traverse_obj(seasons_response, ('items', ..., {dict})): | |
521 | episodes_response = self._call_cms_api_signed( | |
add96eb9 | 522 | f'episodes?season_id={season["id"]}', season['id'], lang, 'episode list') |
032de83e SS |
523 | for episode_response in traverse_obj(episodes_response, ('items', ..., {dict})): |
524 | yield self.url_result( | |
525 | f'{self._BASE_URL}/{lang}watch/{episode_response["id"]}', | |
526 | CrunchyrollBetaIE, **CrunchyrollBetaIE._transform_episode_response(episode_response)) | |
527 | ||
528 | return self.playlist_result( | |
529 | entries(), internal_id, | |
530 | **traverse_obj(self._call_api(f'series/{internal_id}', internal_id, lang, 'series'), ('data', 0, { | |
531 | 'title': ('title', {str}), | |
532 | 'description': ('description', {lambda x: x.replace(r'\r\n', '\n')}), | |
533 | 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), | |
534 | 'thumbnails': ('images', ..., ..., ..., { | |
535 | 'url': ('source', {url_or_none}), | |
536 | 'width': ('width', {int_or_none}), | |
537 | 'height': ('height', {int_or_none}), | |
add96eb9 | 538 | }), |
032de83e SS |
539 | }))) |
540 | ||
541 | ||
542 | class CrunchyrollMusicIE(CrunchyrollBaseIE): | |
543 | IE_NAME = 'crunchyroll:music' | |
544 | _VALID_URL = r'''(?x) | |
545 | https?://(?:www\.)?crunchyroll\.com/ | |
546 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) | |
5b4b9276 | 547 | watch/(?P<type>concert|musicvideo)/(?P<id>\w+)''' |
032de83e | 548 | _TESTS = [{ |
5b4b9276 AS |
549 | 'url': 'https://www.crunchyroll.com/de/watch/musicvideo/MV5B02C79', |
550 | 'info_dict': { | |
551 | 'ext': 'mp4', | |
552 | 'id': 'MV5B02C79', | |
553 | 'display_id': 'egaono-hana', | |
554 | 'title': 'Egaono Hana', | |
555 | 'track': 'Egaono Hana', | |
954e57e4 | 556 | 'artists': ['Goose house'], |
5b4b9276 | 557 | 'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', |
f4f9f6d0 | 558 | 'genres': ['J-Pop'], |
5b4b9276 AS |
559 | }, |
560 | 'params': {'skip_download': 'm3u8'}, | |
561 | }, { | |
032de83e SS |
562 | 'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C', |
563 | 'info_dict': { | |
564 | 'ext': 'mp4', | |
565 | 'id': 'MV88BB7F2C', | |
566 | 'display_id': 'crossing-field', | |
567 | 'title': 'Crossing Field', | |
568 | 'track': 'Crossing Field', | |
954e57e4 | 569 | 'artists': ['LiSA'], |
032de83e | 570 | 'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', |
f4f9f6d0 | 571 | 'genres': ['Anime'], |
032de83e SS |
572 | }, |
573 | 'params': {'skip_download': 'm3u8'}, | |
954e57e4 | 574 | 'skip': 'no longer exists', |
032de83e SS |
575 | }, { |
576 | 'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135', | |
577 | 'info_dict': { | |
578 | 'ext': 'mp4', | |
579 | 'id': 'MC2E2AC135', | |
580 | 'display_id': 'live-is-smile-always-364joker-at-yokohama-arena', | |
581 | 'title': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA', | |
582 | 'track': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA', | |
954e57e4 | 583 | 'artists': ['LiSA'], |
032de83e SS |
584 | 'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', |
585 | 'description': 'md5:747444e7e6300907b7a43f0a0503072e', | |
f4f9f6d0 | 586 | 'genres': ['J-Pop'], |
032de83e SS |
587 | }, |
588 | 'params': {'skip_download': 'm3u8'}, | |
589 | }, { | |
5b4b9276 | 590 | 'url': 'https://www.crunchyroll.com/de/watch/musicvideo/MV5B02C79/egaono-hana', |
032de83e SS |
591 | 'only_matching': True, |
592 | }, { | |
593 | 'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135/live-is-smile-always-364joker-at-yokohama-arena', | |
594 | 'only_matching': True, | |
5b4b9276 AS |
595 | }, { |
596 | 'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field', | |
597 | 'only_matching': True, | |
032de83e SS |
598 | }] |
599 | _API_ENDPOINT = 'music' | |
600 | ||
601 | def _real_extract(self, url): | |
602 | lang, internal_id, object_type = self._match_valid_url(url).group('lang', 'id', 'type') | |
603 | path, name = { | |
604 | 'concert': ('concerts', 'concert info'), | |
605 | 'musicvideo': ('music_videos', 'music video info'), | |
606 | }[object_type] | |
607 | response = traverse_obj(self._call_api(f'{path}/{internal_id}', internal_id, lang, name), ('data', 0, {dict})) | |
608 | if not response: | |
609 | raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True) | |
f4d706a9 | 610 | |
cb2fb4a6 | 611 | result = self._transform_music_response(response) |
612 | ||
954e57e4 | 613 | if not self._IS_PREMIUM and response.get('isPremiumOnly'): |
032de83e | 614 | message = f'This {response.get("type") or "media"} is for premium members only' |
ff38a011 | 615 | if CrunchyrollBaseIE._REFRESH_TOKEN: |
cb2fb4a6 | 616 | self.raise_no_formats(message, expected=True, video_id=internal_id) |
617 | else: | |
618 | self.raise_login_required(message, method='password', metadata_available=True) | |
619 | else: | |
620 | result['formats'], _ = self._extract_stream(f'music/{internal_id}', internal_id) | |
032de83e SS |
621 | |
622 | return result | |
623 | ||
624 | @staticmethod | |
625 | def _transform_music_response(data): | |
626 | return { | |
627 | 'id': data['id'], | |
628 | **traverse_obj(data, { | |
629 | 'display_id': 'slug', | |
630 | 'title': 'title', | |
631 | 'track': 'title', | |
954e57e4 | 632 | 'artists': ('artist', 'name', all), |
032de83e SS |
633 | 'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n') or None}), |
634 | 'thumbnails': ('images', ..., ..., { | |
635 | 'url': ('source', {url_or_none}), | |
636 | 'width': ('width', {int_or_none}), | |
637 | 'height': ('height', {int_or_none}), | |
638 | }), | |
f4f9f6d0 | 639 | 'genres': ('genres', ..., 'displayValue'), |
032de83e SS |
640 | 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), |
641 | }), | |
642 | } | |
f4d706a9 | 643 | |
032de83e SS |
644 | |
645 | class CrunchyrollArtistIE(CrunchyrollBaseIE): | |
646 | IE_NAME = 'crunchyroll:artist' | |
647 | _VALID_URL = r'''(?x) | |
648 | https?://(?:www\.)?crunchyroll\.com/ | |
649 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) | |
650 | artist/(?P<id>\w{10})''' | |
651 | _TESTS = [{ | |
652 | 'url': 'https://www.crunchyroll.com/artist/MA179CB50D', | |
653 | 'info_dict': { | |
654 | 'id': 'MA179CB50D', | |
655 | 'title': 'LiSA', | |
954e57e4 | 656 | 'genres': ['Anime', 'J-Pop', 'Rock'], |
032de83e SS |
657 | 'description': 'md5:16d87de61a55c3f7d6c454b73285938e', |
658 | }, | |
659 | 'playlist_mincount': 83, | |
660 | }, { | |
661 | 'url': 'https://www.crunchyroll.com/artist/MA179CB50D/lisa', | |
662 | 'only_matching': True, | |
663 | }] | |
664 | _API_ENDPOINT = 'music' | |
665 | ||
666 | def _real_extract(self, url): | |
667 | lang, internal_id = self._match_valid_url(url).group('lang', 'id') | |
668 | response = traverse_obj(self._call_api( | |
669 | f'artists/{internal_id}', internal_id, lang, 'artist info'), ('data', 0)) | |
f4d706a9 JH |
670 | |
671 | def entries(): | |
032de83e SS |
672 | for attribute, path in [('concerts', 'concert'), ('videos', 'musicvideo')]: |
673 | for internal_id in traverse_obj(response, (attribute, ...)): | |
674 | yield self.url_result(f'{self._BASE_URL}/watch/{path}/{internal_id}', CrunchyrollMusicIE, internal_id) | |
675 | ||
676 | return self.playlist_result(entries(), **self._transform_artist_response(response)) | |
677 | ||
678 | @staticmethod | |
679 | def _transform_artist_response(data): | |
680 | return { | |
681 | 'id': data['id'], | |
682 | **traverse_obj(data, { | |
683 | 'title': 'name', | |
684 | 'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}), | |
685 | 'thumbnails': ('images', ..., ..., { | |
686 | 'url': ('source', {url_or_none}), | |
687 | 'width': ('width', {int_or_none}), | |
688 | 'height': ('height', {int_or_none}), | |
689 | }), | |
f4f9f6d0 | 690 | 'genres': ('genres', ..., 'displayValue'), |
032de83e SS |
691 | }), |
692 | } |