]>
Commit | Line | Data |
---|---|---|
706dfe44 | 1 | import base64 |
032de83e | 2 | import urllib.error |
ac668111 | 3 | |
46279958 | 4 | from .common import InfoExtractor |
1cc79574 PH |
5 | from ..utils import ( |
6 | ExtractorError, | |
54a5be4d | 7 | float_or_none, |
706dfe44 | 8 | format_field, |
032de83e | 9 | int_or_none, |
706dfe44 | 10 | join_nonempty, |
032de83e SS |
11 | parse_age_limit, |
12 | parse_count, | |
b99ba3df | 13 | parse_iso8601, |
a9d4da60 | 14 | qualities, |
032de83e SS |
15 | remove_start, |
16 | time_seconds, | |
706dfe44 | 17 | traverse_obj, |
032de83e SS |
18 | url_or_none, |
19 | urlencode_postdata, | |
c8434e83 | 20 | ) |
c8434e83 | 21 | |
34440095 | 22 | |
46279958 | 23 | class CrunchyrollBaseIE(InfoExtractor): |
032de83e | 24 | _BASE_URL = 'https://www.crunchyroll.com' |
7c74a015 | 25 | _API_BASE = 'https://api.crunchyroll.com' |
80f48920 | 26 | _NETRC_MACHINE = 'crunchyroll' |
032de83e SS |
27 | _AUTH_HEADERS = None |
28 | _API_ENDPOINT = None | |
29 | _BASIC_AUTH = None | |
30 | _QUERY = {} | |
05dee6c5 | 31 | |
44699d10 | 32 | @property |
33 | def is_logged_in(self): | |
032de83e | 34 | return self._get_cookies(self._BASE_URL).get('etp_rt') |
44699d10 | 35 | |
52efa4b3 | 36 | def _perform_login(self, username, password): |
44699d10 | 37 | if self.is_logged_in: |
eb5b1fc0 S |
38 | return |
39 | ||
7c74a015 JH |
40 | upsell_response = self._download_json( |
41 | f'{self._API_BASE}/get_upsell_data.0.json', None, 'Getting session id', | |
42 | query={ | |
43 | 'sess_id': 1, | |
44 | 'device_id': 'whatvalueshouldbeforweb', | |
45 | 'device_type': 'com.crunchyroll.static', | |
46 | 'access_token': 'giKq5eY27ny3cqz', | |
032de83e | 47 | 'referer': f'{self._BASE_URL}/welcome/login' |
7c74a015 JH |
48 | }) |
49 | if upsell_response['code'] != 'ok': | |
50 | raise ExtractorError('Could not get session id') | |
51 | session_id = upsell_response['data']['session_id'] | |
52 | ||
53 | login_response = self._download_json( | |
54 | f'{self._API_BASE}/login.1.json', None, 'Logging in', | |
032de83e | 55 | data=urlencode_postdata({ |
7c74a015 JH |
56 | 'account': username, |
57 | 'password': password, | |
58 | 'session_id': session_id | |
032de83e | 59 | })) |
7c74a015 | 60 | if login_response['code'] != 'ok': |
97bef011 | 61 | raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True) |
44699d10 | 62 | if not self.is_logged_in: |
7c74a015 | 63 | raise ExtractorError('Login succeeded but did not set etp_rt cookie') |
80f48920 | 64 | |
032de83e SS |
65 | def _update_query(self, lang): |
66 | if lang in CrunchyrollBaseIE._QUERY: | |
67 | return | |
68 | ||
69 | webpage = self._download_webpage( | |
70 | f'{self._BASE_URL}/{lang}', None, note=f'Retrieving main page (lang={lang or None})') | |
71 | ||
72 | initial_state = self._search_json(r'__INITIAL_STATE__\s*=', webpage, 'initial state', None) | |
73 | CrunchyrollBaseIE._QUERY[lang] = traverse_obj(initial_state, { | |
74 | 'locale': ('localization', 'locale'), | |
75 | }) or None | |
76 | ||
77 | if CrunchyrollBaseIE._BASIC_AUTH: | |
78 | return | |
79 | ||
80 | app_config = self._search_json(r'__APP_CONFIG__\s*=', webpage, 'app config', None) | |
81 | cx_api_param = app_config['cxApiParams']['accountAuthClientId' if self.is_logged_in else 'anonClientId'] | |
82 | self.write_debug(f'Using cxApiParam={cx_api_param}') | |
83 | CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode() | |
84 | ||
85 | def _update_auth(self): | |
86 | if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_REFRESH > time_seconds(): | |
87 | return | |
88 | ||
89 | assert CrunchyrollBaseIE._BASIC_AUTH, '_update_query needs to be called at least one time beforehand' | |
90 | grant_type = 'etp_rt_cookie' if self.is_logged_in else 'client_id' | |
91 | auth_response = self._download_json( | |
92 | f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}', | |
93 | headers={'Authorization': CrunchyrollBaseIE._BASIC_AUTH}, data=f'grant_type={grant_type}'.encode()) | |
94 | ||
95 | CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']} | |
96 | CrunchyrollBaseIE._AUTH_REFRESH = time_seconds(seconds=traverse_obj(auth_response, ('expires_in', {float_or_none}), default=300) - 10) | |
97 | ||
98 | def _call_base_api(self, endpoint, internal_id, lang, note=None, query={}): | |
99 | self._update_query(lang) | |
100 | self._update_auth() | |
101 | ||
102 | if not endpoint.startswith('/'): | |
103 | endpoint = f'/{endpoint}' | |
104 | ||
105 | return self._download_json( | |
106 | f'{self._BASE_URL}{endpoint}', internal_id, note or f'Calling API: {endpoint}', | |
107 | headers=CrunchyrollBaseIE._AUTH_HEADERS, query={**CrunchyrollBaseIE._QUERY[lang], **query}) | |
108 | ||
109 | def _call_api(self, path, internal_id, lang, note='api', query={}): | |
110 | if not path.startswith(f'/content/v2/{self._API_ENDPOINT}/'): | |
111 | path = f'/content/v2/{self._API_ENDPOINT}/{path}' | |
112 | ||
113 | try: | |
114 | result = self._call_base_api( | |
115 | path, internal_id, lang, f'Downloading {note} JSON ({self._API_ENDPOINT})', query=query) | |
116 | except ExtractorError as error: | |
117 | if isinstance(error.cause, urllib.error.HTTPError) and error.cause.code == 404: | |
118 | return None | |
119 | raise | |
120 | ||
121 | if not result: | |
122 | raise ExtractorError(f'Unexpected response when downloading {note} JSON') | |
123 | return result | |
124 | ||
125 | def _extract_formats(self, stream_response, display_id=None): | |
126 | requested_formats = self._configuration_arg('format') or ['adaptive_hls'] | |
127 | available_formats = {} | |
128 | for stream_type, streams in traverse_obj( | |
129 | stream_response, (('streams', ('data', 0)), {dict.items}, ...)): | |
130 | if stream_type not in requested_formats: | |
131 | continue | |
132 | for stream in traverse_obj(streams, lambda _, v: v['url']): | |
133 | hardsub_lang = stream.get('hardsub_locale') or '' | |
134 | format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s')) | |
135 | available_formats[hardsub_lang] = (stream_type, format_id, hardsub_lang, stream['url']) | |
136 | ||
137 | requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])] | |
138 | if '' in available_formats and 'all' not in requested_hardsubs: | |
139 | full_format_langs = set(requested_hardsubs) | |
140 | self.to_screen( | |
141 | 'To get all formats of a hardsub language, use ' | |
142 | '"--extractor-args crunchyrollbeta:hardsub=<language_code or all>". ' | |
143 | 'See https://github.com/yt-dlp/yt-dlp#crunchyrollbeta-crunchyroll for more info', | |
144 | only_once=True) | |
145 | else: | |
146 | full_format_langs = set(map(str.lower, available_formats)) | |
147 | ||
148 | audio_locale = traverse_obj(stream_response, ((None, 'meta'), 'audio_locale'), get_all=False) | |
149 | hardsub_preference = qualities(requested_hardsubs[::-1]) | |
150 | formats = [] | |
151 | for stream_type, format_id, hardsub_lang, stream_url in available_formats.values(): | |
152 | if stream_type.endswith('hls'): | |
153 | if hardsub_lang.lower() in full_format_langs: | |
154 | adaptive_formats = self._extract_m3u8_formats( | |
155 | stream_url, display_id, 'mp4', m3u8_id=format_id, | |
156 | fatal=False, note=f'Downloading {format_id} HLS manifest') | |
157 | else: | |
158 | adaptive_formats = (self._m3u8_meta_format(stream_url, ext='mp4', m3u8_id=format_id),) | |
159 | elif stream_type.endswith('dash'): | |
160 | adaptive_formats = self._extract_mpd_formats( | |
161 | stream_url, display_id, mpd_id=format_id, | |
162 | fatal=False, note=f'Downloading {format_id} MPD manifest') | |
459262ac | 163 | else: |
032de83e SS |
164 | self.report_warning(f'Encountered unknown stream_type: {stream_type!r}', display_id, only_once=True) |
165 | continue | |
166 | for f in adaptive_formats: | |
167 | if f.get('acodec') != 'none': | |
168 | f['language'] = audio_locale | |
169 | f['quality'] = hardsub_preference(hardsub_lang.lower()) | |
170 | formats.extend(adaptive_formats) | |
171 | ||
172 | return formats | |
173 | ||
174 | def _extract_subtitles(self, data): | |
175 | subtitles = {} | |
176 | ||
177 | for locale, subtitle in traverse_obj(data, ((None, 'meta'), 'subtitles', {dict.items}, ...)): | |
178 | subtitles[locale] = [traverse_obj(subtitle, {'url': 'url', 'ext': 'format'})] | |
179 | ||
180 | return subtitles | |
181 | ||
182 | ||
183 | class CrunchyrollCmsBaseIE(CrunchyrollBaseIE): | |
184 | _API_ENDPOINT = 'cms' | |
185 | _CMS_EXPIRY = None | |
186 | ||
187 | def _call_cms_api_signed(self, path, internal_id, lang, note='api'): | |
188 | if not CrunchyrollCmsBaseIE._CMS_EXPIRY or CrunchyrollCmsBaseIE._CMS_EXPIRY <= time_seconds(): | |
189 | response = self._call_base_api('index/v2', None, lang, 'Retrieving signed policy')['cms_web'] | |
190 | CrunchyrollCmsBaseIE._CMS_QUERY = { | |
191 | 'Policy': response['policy'], | |
192 | 'Signature': response['signature'], | |
193 | 'Key-Pair-Id': response['key_pair_id'], | |
f4d706a9 | 194 | } |
032de83e SS |
195 | CrunchyrollCmsBaseIE._CMS_BUCKET = response['bucket'] |
196 | CrunchyrollCmsBaseIE._CMS_EXPIRY = parse_iso8601(response['expires']) - 10 | |
197 | ||
198 | if not path.startswith('/cms/v2'): | |
199 | path = f'/cms/v2{CrunchyrollCmsBaseIE._CMS_BUCKET}/{path}' | |
f4d706a9 | 200 | |
032de83e SS |
201 | return self._call_base_api( |
202 | path, internal_id, lang, f'Downloading {note} JSON (signed cms)', query=CrunchyrollCmsBaseIE._CMS_QUERY) | |
f4d706a9 | 203 | |
032de83e SS |
204 | |
205 | class CrunchyrollBetaIE(CrunchyrollCmsBaseIE): | |
cb1553e9 | 206 | IE_NAME = 'crunchyroll' |
5da42f2b | 207 | _VALID_URL = r'''(?x) |
032de83e | 208 | https?://(?:beta\.|www\.)?crunchyroll\.com/ |
5da42f2b | 209 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) |
032de83e | 210 | watch/(?!concert|musicvideo)(?P<id>\w+)''' |
dd078970 | 211 | _TESTS = [{ |
032de83e | 212 | # Premium only |
cb1553e9 | 213 | 'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future', |
dd078970 | 214 | 'info_dict': { |
b99ba3df | 215 | 'id': 'GY2P1Q98Y', |
dd078970 | 216 | 'ext': 'mp4', |
b99ba3df JH |
217 | 'duration': 1380.241, |
218 | 'timestamp': 1459632600, | |
dd078970 | 219 | 'description': 'md5:a022fbec4fbb023d43631032c91ed64b', |
dd078970 | 220 | 'title': 'World Trigger Episode 73 – To the Future', |
221 | 'upload_date': '20160402', | |
f4d706a9 | 222 | 'series': 'World Trigger', |
b99ba3df | 223 | 'series_id': 'GR757DMKY', |
f4d706a9 | 224 | 'season': 'World Trigger', |
b99ba3df | 225 | 'season_id': 'GR9P39NJ6', |
f4d706a9 | 226 | 'season_number': 1, |
b99ba3df JH |
227 | 'episode': 'To the Future', |
228 | 'episode_number': 73, | |
032de83e | 229 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', |
93abb740 | 230 | 'chapters': 'count:2', |
032de83e SS |
231 | 'age_limit': 14, |
232 | 'like_count': int, | |
233 | 'dislike_count': int, | |
dd078970 | 234 | }, |
dfea94f8 SS |
235 | 'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'}, |
236 | }, { | |
032de83e | 237 | # Premium only |
cb1553e9 | 238 | 'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR', |
dfea94f8 SS |
239 | 'info_dict': { |
240 | 'id': 'GYE5WKQGR', | |
241 | 'ext': 'mp4', | |
242 | 'duration': 366.459, | |
243 | 'timestamp': 1476788400, | |
244 | 'description': 'md5:74b67283ffddd75f6e224ca7dc031e76', | |
032de83e | 245 | 'title': 'SHELTER – Porter Robinson presents Shelter the Animation', |
dfea94f8 SS |
246 | 'upload_date': '20161018', |
247 | 'series': 'SHELTER', | |
248 | 'series_id': 'GYGG09WWY', | |
249 | 'season': 'SHELTER', | |
250 | 'season_id': 'GR09MGK4R', | |
251 | 'season_number': 1, | |
252 | 'episode': 'Porter Robinson presents Shelter the Animation', | |
253 | 'episode_number': 0, | |
032de83e SS |
254 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', |
255 | 'age_limit': 14, | |
256 | 'like_count': int, | |
257 | 'dislike_count': int, | |
dfea94f8 SS |
258 | }, |
259 | 'params': {'skip_download': True}, | |
032de83e SS |
260 | }, { |
261 | 'url': 'https://www.crunchyroll.com/watch/GJWU2VKK3/cherry-blossom-meeting-and-a-coming-blizzard', | |
262 | 'info_dict': { | |
263 | 'id': 'GJWU2VKK3', | |
264 | 'ext': 'mp4', | |
265 | 'duration': 1420.054, | |
266 | 'description': 'md5:2d1c67c0ec6ae514d9c30b0b99a625cd', | |
267 | 'title': 'The Ice Guy and His Cool Female Colleague Episode 1 – Cherry Blossom Meeting and a Coming Blizzard', | |
268 | 'series': 'The Ice Guy and His Cool Female Colleague', | |
269 | 'series_id': 'GW4HM75NP', | |
270 | 'season': 'The Ice Guy and His Cool Female Colleague', | |
271 | 'season_id': 'GY9PC21VE', | |
272 | 'season_number': 1, | |
273 | 'episode': 'Cherry Blossom Meeting and a Coming Blizzard', | |
274 | 'episode_number': 1, | |
275 | 'chapters': 'count:2', | |
276 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', | |
277 | 'timestamp': 1672839000, | |
278 | 'upload_date': '20230104', | |
279 | 'age_limit': 14, | |
280 | 'like_count': int, | |
281 | 'dislike_count': int, | |
282 | }, | |
283 | 'params': {'skip_download': 'm3u8'}, | |
284 | }, { | |
285 | 'url': 'https://www.crunchyroll.com/watch/GM8F313NQ', | |
286 | 'info_dict': { | |
287 | 'id': 'GM8F313NQ', | |
288 | 'ext': 'mp4', | |
289 | 'title': 'Garakowa -Restore the World-', | |
290 | 'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608', | |
291 | 'duration': 3996.104, | |
292 | 'age_limit': 13, | |
293 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', | |
294 | }, | |
295 | 'params': {'skip_download': 'm3u8'}, | |
296 | }, { | |
297 | 'url': 'https://www.crunchyroll.com/watch/G62PEZ2E6', | |
298 | 'info_dict': { | |
299 | 'id': 'G62PEZ2E6', | |
300 | 'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608', | |
301 | 'age_limit': 13, | |
302 | 'duration': 65.138, | |
303 | 'title': 'Garakowa -Restore the World-', | |
304 | }, | |
305 | 'playlist_mincount': 5, | |
f4d706a9 | 306 | }, { |
cb1553e9 | 307 | 'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y', |
f4d706a9 | 308 | 'only_matching': True, |
964b5493 | 309 | }, { |
310 | 'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy', | |
311 | 'only_matching': True, | |
dd078970 | 312 | }] |
032de83e SS |
313 | # We want to support lazy playlist filtering and movie listings cannot be inside a playlist |
314 | _RETURN_TYPE = 'video' | |
dd078970 | 315 | |
316 | def _real_extract(self, url): | |
032de83e | 317 | lang, internal_id = self._match_valid_url(url).group('lang', 'id') |
f4d706a9 | 318 | |
032de83e SS |
319 | # We need to use unsigned API call to allow ratings query string |
320 | response = traverse_obj(self._call_api( | |
321 | f'objects/{internal_id}', internal_id, lang, 'object info', {'ratings': 'true'}), ('data', 0, {dict})) | |
322 | if not response: | |
323 | raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True) | |
706dfe44 | 324 | |
032de83e SS |
325 | object_type = response.get('type') |
326 | if object_type == 'episode': | |
327 | result = self._transform_episode_response(response) | |
706dfe44 | 328 | |
032de83e SS |
329 | elif object_type == 'movie': |
330 | result = self._transform_movie_response(response) | |
706dfe44 | 331 | |
032de83e SS |
332 | elif object_type == 'movie_listing': |
333 | first_movie_id = traverse_obj(response, ('movie_listing_metadata', 'first_movie_id')) | |
334 | if not self._yes_playlist(internal_id, first_movie_id): | |
335 | return self.url_result(f'{self._BASE_URL}/{lang}watch/{first_movie_id}', CrunchyrollBetaIE, first_movie_id) | |
336 | ||
337 | def entries(): | |
338 | movies = self._call_api(f'movie_listings/{internal_id}/movies', internal_id, lang, 'movie list') | |
339 | for movie_response in traverse_obj(movies, ('data', ...)): | |
340 | yield self.url_result( | |
341 | f'{self._BASE_URL}/{lang}watch/{movie_response["id"]}', | |
342 | CrunchyrollBetaIE, **self._transform_movie_response(movie_response)) | |
343 | ||
344 | return self.playlist_result(entries(), **self._transform_movie_response(response)) | |
dfea94f8 | 345 | |
dfea94f8 | 346 | else: |
032de83e | 347 | raise ExtractorError(f'Unknown object type {object_type}') |
dfea94f8 | 348 | |
032de83e SS |
349 | # There might be multiple audio languages for one object (`<object>_metadata.versions`), |
350 | # so we need to get the id from `streams_link` instead or we dont know which language to choose | |
351 | streams_link = response.get('streams_link') | |
352 | if not streams_link and traverse_obj(response, (f'{object_type}_metadata', 'is_premium_only')): | |
353 | message = f'This {object_type} is for premium members only' | |
354 | if self.is_logged_in: | |
355 | raise ExtractorError(message, expected=True) | |
356 | self.raise_login_required(message) | |
357 | ||
358 | # We need go from unsigned to signed api to avoid getting soft banned | |
359 | stream_response = self._call_cms_api_signed(remove_start( | |
360 | streams_link, '/content/v2/cms/'), internal_id, lang, 'stream info') | |
361 | result['formats'] = self._extract_formats(stream_response, internal_id) | |
362 | result['subtitles'] = self._extract_subtitles(stream_response) | |
706dfe44 | 363 | |
93abb740 | 364 | # if no intro chapter is available, a 403 without usable data is returned |
032de83e SS |
365 | intro_chapter = self._download_json( |
366 | f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json', | |
367 | internal_id, note='Downloading chapter info', fatal=False, errnote=False) | |
93abb740 | 368 | if isinstance(intro_chapter, dict): |
032de83e | 369 | result['chapters'] = [{ |
93abb740 B |
370 | 'title': 'Intro', |
371 | 'start_time': float_or_none(intro_chapter.get('startTime')), | |
032de83e | 372 | 'end_time': float_or_none(intro_chapter.get('endTime')), |
93abb740 B |
373 | }] |
374 | ||
032de83e SS |
375 | def calculate_count(item): |
376 | return parse_count(''.join((item['displayed'], item.get('unit') or ''))) | |
377 | ||
378 | result.update(traverse_obj(response, ('rating', { | |
379 | 'like_count': ('up', {calculate_count}), | |
380 | 'dislike_count': ('down', {calculate_count}), | |
381 | }))) | |
382 | ||
383 | return result | |
384 | ||
385 | @staticmethod | |
386 | def _transform_episode_response(data): | |
387 | metadata = traverse_obj(data, (('episode_metadata', None), {dict}), get_all=False) or {} | |
706dfe44 | 388 | return { |
032de83e SS |
389 | 'id': data['id'], |
390 | 'title': ' \u2013 '.join(( | |
391 | ('%s%s' % ( | |
392 | format_field(metadata, 'season_title'), | |
393 | format_field(metadata, 'episode', ' Episode %s'))), | |
394 | format_field(data, 'title'))), | |
395 | **traverse_obj(data, { | |
396 | 'episode': ('title', {str}), | |
397 | 'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}), | |
398 | 'thumbnails': ('images', 'thumbnail', ..., ..., { | |
399 | 'url': ('source', {url_or_none}), | |
400 | 'width': ('width', {int_or_none}), | |
401 | 'height': ('height', {int_or_none}), | |
402 | }), | |
403 | }), | |
404 | **traverse_obj(metadata, { | |
405 | 'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}), | |
406 | 'timestamp': ('upload_date', {parse_iso8601}), | |
407 | 'series': ('series_title', {str}), | |
408 | 'series_id': ('series_id', {str}), | |
409 | 'season': ('season_title', {str}), | |
410 | 'season_id': ('season_id', {str}), | |
411 | 'season_number': ('season_number', ({int}, {float_or_none})), | |
412 | 'episode_number': ('sequence_number', ({int}, {float_or_none})), | |
413 | 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), | |
414 | 'language': ('audio_locale', {str}), | |
415 | }, get_all=False), | |
706dfe44 | 416 | } |
dd078970 | 417 | |
032de83e SS |
418 | @staticmethod |
419 | def _transform_movie_response(data): | |
420 | metadata = traverse_obj(data, (('movie_metadata', 'movie_listing_metadata', None), {dict}), get_all=False) or {} | |
421 | return { | |
422 | 'id': data['id'], | |
423 | **traverse_obj(data, { | |
424 | 'title': ('title', {str}), | |
425 | 'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}), | |
426 | 'thumbnails': ('images', 'thumbnail', ..., ..., { | |
427 | 'url': ('source', {url_or_none}), | |
428 | 'width': ('width', {int_or_none}), | |
429 | 'height': ('height', {int_or_none}), | |
430 | }), | |
431 | }), | |
432 | **traverse_obj(metadata, { | |
433 | 'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}), | |
434 | 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), | |
435 | }), | |
436 | } | |
dd078970 | 437 | |
032de83e SS |
438 | |
439 | class CrunchyrollBetaShowIE(CrunchyrollCmsBaseIE): | |
cb1553e9 | 440 | IE_NAME = 'crunchyroll:playlist' |
5da42f2b | 441 | _VALID_URL = r'''(?x) |
032de83e | 442 | https?://(?:beta\.|www\.)?crunchyroll\.com/ |
5da42f2b | 443 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) |
032de83e | 444 | series/(?P<id>\w+)''' |
dd078970 | 445 | _TESTS = [{ |
cb1553e9 | 446 | 'url': 'https://www.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA', |
dd078970 | 447 | 'info_dict': { |
b99ba3df | 448 | 'id': 'GY19NQ2QR', |
dd078970 | 449 | 'title': 'Girl Friend BETA', |
032de83e SS |
450 | 'description': 'md5:99c1b22ee30a74b536a8277ced8eb750', |
451 | # XXX: `thumbnail` does not get set from `thumbnails` in playlist | |
452 | # 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', | |
453 | 'age_limit': 14, | |
dd078970 | 454 | }, |
455 | 'playlist_mincount': 10, | |
456 | }, { | |
5da42f2b | 457 | 'url': 'https://beta.crunchyroll.com/it/series/GY19NQ2QR', |
dd078970 | 458 | 'only_matching': True, |
459 | }] | |
460 | ||
461 | def _real_extract(self, url): | |
032de83e SS |
462 | lang, internal_id = self._match_valid_url(url).group('lang', 'id') |
463 | ||
464 | def entries(): | |
465 | seasons_response = self._call_cms_api_signed(f'seasons?series_id={internal_id}', internal_id, lang, 'seasons') | |
466 | for season in traverse_obj(seasons_response, ('items', ..., {dict})): | |
467 | episodes_response = self._call_cms_api_signed( | |
468 | f'episodes?season_id={season["id"]}', season["id"], lang, 'episode list') | |
469 | for episode_response in traverse_obj(episodes_response, ('items', ..., {dict})): | |
470 | yield self.url_result( | |
471 | f'{self._BASE_URL}/{lang}watch/{episode_response["id"]}', | |
472 | CrunchyrollBetaIE, **CrunchyrollBetaIE._transform_episode_response(episode_response)) | |
473 | ||
474 | return self.playlist_result( | |
475 | entries(), internal_id, | |
476 | **traverse_obj(self._call_api(f'series/{internal_id}', internal_id, lang, 'series'), ('data', 0, { | |
477 | 'title': ('title', {str}), | |
478 | 'description': ('description', {lambda x: x.replace(r'\r\n', '\n')}), | |
479 | 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), | |
480 | 'thumbnails': ('images', ..., ..., ..., { | |
481 | 'url': ('source', {url_or_none}), | |
482 | 'width': ('width', {int_or_none}), | |
483 | 'height': ('height', {int_or_none}), | |
484 | }) | |
485 | }))) | |
486 | ||
487 | ||
488 | class CrunchyrollMusicIE(CrunchyrollBaseIE): | |
489 | IE_NAME = 'crunchyroll:music' | |
490 | _VALID_URL = r'''(?x) | |
491 | https?://(?:www\.)?crunchyroll\.com/ | |
492 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) | |
493 | watch/(?P<type>concert|musicvideo)/(?P<id>\w{10})''' | |
494 | _TESTS = [{ | |
495 | 'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C', | |
496 | 'info_dict': { | |
497 | 'ext': 'mp4', | |
498 | 'id': 'MV88BB7F2C', | |
499 | 'display_id': 'crossing-field', | |
500 | 'title': 'Crossing Field', | |
501 | 'track': 'Crossing Field', | |
502 | 'artist': 'LiSA', | |
503 | 'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', | |
504 | 'genre': ['Anime'], | |
505 | }, | |
506 | 'params': {'skip_download': 'm3u8'}, | |
507 | }, { | |
508 | 'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135', | |
509 | 'info_dict': { | |
510 | 'ext': 'mp4', | |
511 | 'id': 'MC2E2AC135', | |
512 | 'display_id': 'live-is-smile-always-364joker-at-yokohama-arena', | |
513 | 'title': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA', | |
514 | 'track': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA', | |
515 | 'artist': 'LiSA', | |
516 | 'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', | |
517 | 'description': 'md5:747444e7e6300907b7a43f0a0503072e', | |
518 | 'genre': ['J-Pop'], | |
519 | }, | |
520 | 'params': {'skip_download': 'm3u8'}, | |
521 | }, { | |
522 | 'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field', | |
523 | 'only_matching': True, | |
524 | }, { | |
525 | 'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135/live-is-smile-always-364joker-at-yokohama-arena', | |
526 | 'only_matching': True, | |
527 | }] | |
528 | _API_ENDPOINT = 'music' | |
529 | ||
530 | def _real_extract(self, url): | |
531 | lang, internal_id, object_type = self._match_valid_url(url).group('lang', 'id', 'type') | |
532 | path, name = { | |
533 | 'concert': ('concerts', 'concert info'), | |
534 | 'musicvideo': ('music_videos', 'music video info'), | |
535 | }[object_type] | |
536 | response = traverse_obj(self._call_api(f'{path}/{internal_id}', internal_id, lang, name), ('data', 0, {dict})) | |
537 | if not response: | |
538 | raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True) | |
f4d706a9 | 539 | |
032de83e SS |
540 | streams_link = response.get('streams_link') |
541 | if not streams_link and response.get('isPremiumOnly'): | |
542 | message = f'This {response.get("type") or "media"} is for premium members only' | |
543 | if self.is_logged_in: | |
544 | raise ExtractorError(message, expected=True) | |
545 | self.raise_login_required(message) | |
546 | ||
547 | result = self._transform_music_response(response) | |
548 | stream_response = self._call_api(streams_link, internal_id, lang, 'stream info') | |
549 | result['formats'] = self._extract_formats(stream_response, internal_id) | |
550 | ||
551 | return result | |
552 | ||
553 | @staticmethod | |
554 | def _transform_music_response(data): | |
555 | return { | |
556 | 'id': data['id'], | |
557 | **traverse_obj(data, { | |
558 | 'display_id': 'slug', | |
559 | 'title': 'title', | |
560 | 'track': 'title', | |
561 | 'artist': ('artist', 'name'), | |
562 | 'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n') or None}), | |
563 | 'thumbnails': ('images', ..., ..., { | |
564 | 'url': ('source', {url_or_none}), | |
565 | 'width': ('width', {int_or_none}), | |
566 | 'height': ('height', {int_or_none}), | |
567 | }), | |
568 | 'genre': ('genres', ..., 'displayValue'), | |
569 | 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), | |
570 | }), | |
571 | } | |
f4d706a9 | 572 | |
032de83e SS |
573 | |
574 | class CrunchyrollArtistIE(CrunchyrollBaseIE): | |
575 | IE_NAME = 'crunchyroll:artist' | |
576 | _VALID_URL = r'''(?x) | |
577 | https?://(?:www\.)?crunchyroll\.com/ | |
578 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) | |
579 | artist/(?P<id>\w{10})''' | |
580 | _TESTS = [{ | |
581 | 'url': 'https://www.crunchyroll.com/artist/MA179CB50D', | |
582 | 'info_dict': { | |
583 | 'id': 'MA179CB50D', | |
584 | 'title': 'LiSA', | |
585 | 'genre': ['J-Pop', 'Anime', 'Rock'], | |
586 | 'description': 'md5:16d87de61a55c3f7d6c454b73285938e', | |
587 | }, | |
588 | 'playlist_mincount': 83, | |
589 | }, { | |
590 | 'url': 'https://www.crunchyroll.com/artist/MA179CB50D/lisa', | |
591 | 'only_matching': True, | |
592 | }] | |
593 | _API_ENDPOINT = 'music' | |
594 | ||
595 | def _real_extract(self, url): | |
596 | lang, internal_id = self._match_valid_url(url).group('lang', 'id') | |
597 | response = traverse_obj(self._call_api( | |
598 | f'artists/{internal_id}', internal_id, lang, 'artist info'), ('data', 0)) | |
f4d706a9 JH |
599 | |
600 | def entries(): | |
032de83e SS |
601 | for attribute, path in [('concerts', 'concert'), ('videos', 'musicvideo')]: |
602 | for internal_id in traverse_obj(response, (attribute, ...)): | |
603 | yield self.url_result(f'{self._BASE_URL}/watch/{path}/{internal_id}', CrunchyrollMusicIE, internal_id) | |
604 | ||
605 | return self.playlist_result(entries(), **self._transform_artist_response(response)) | |
606 | ||
607 | @staticmethod | |
608 | def _transform_artist_response(data): | |
609 | return { | |
610 | 'id': data['id'], | |
611 | **traverse_obj(data, { | |
612 | 'title': 'name', | |
613 | 'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}), | |
614 | 'thumbnails': ('images', ..., ..., { | |
615 | 'url': ('source', {url_or_none}), | |
616 | 'width': ('width', {int_or_none}), | |
617 | 'height': ('height', {int_or_none}), | |
618 | }), | |
619 | 'genre': ('genres', ..., 'displayValue'), | |
620 | }), | |
621 | } |