]>
Commit | Line | Data |
---|---|---|
706dfe44 | 1 | import base64 |
cb1553e9 | 2 | import urllib.parse |
ac668111 | 3 | |
46279958 | 4 | from .common import InfoExtractor |
1cc79574 PH |
5 | from ..utils import ( |
6 | ExtractorError, | |
54a5be4d | 7 | float_or_none, |
706dfe44 | 8 | format_field, |
706dfe44 | 9 | join_nonempty, |
b99ba3df | 10 | parse_iso8601, |
a9d4da60 | 11 | qualities, |
706dfe44 | 12 | traverse_obj, |
245d43ca | 13 | try_get, |
c8434e83 | 14 | ) |
c8434e83 | 15 | |
34440095 | 16 | |
46279958 | 17 | class CrunchyrollBaseIE(InfoExtractor): |
7c74a015 JH |
18 | _LOGIN_URL = 'https://www.crunchyroll.com/welcome/login' |
19 | _API_BASE = 'https://api.crunchyroll.com' | |
80f48920 | 20 | _NETRC_MACHINE = 'crunchyroll' |
cb1553e9 | 21 | params = None |
05dee6c5 | 22 | |
44699d10 | 23 | @property |
24 | def is_logged_in(self): | |
25 | return self._get_cookies(self._LOGIN_URL).get('etp_rt') | |
26 | ||
52efa4b3 | 27 | def _perform_login(self, username, password): |
44699d10 | 28 | if self.is_logged_in: |
eb5b1fc0 S |
29 | return |
30 | ||
7c74a015 JH |
31 | upsell_response = self._download_json( |
32 | f'{self._API_BASE}/get_upsell_data.0.json', None, 'Getting session id', | |
33 | query={ | |
34 | 'sess_id': 1, | |
35 | 'device_id': 'whatvalueshouldbeforweb', | |
36 | 'device_type': 'com.crunchyroll.static', | |
37 | 'access_token': 'giKq5eY27ny3cqz', | |
38 | 'referer': self._LOGIN_URL | |
39 | }) | |
40 | if upsell_response['code'] != 'ok': | |
41 | raise ExtractorError('Could not get session id') | |
42 | session_id = upsell_response['data']['session_id'] | |
43 | ||
44 | login_response = self._download_json( | |
45 | f'{self._API_BASE}/login.1.json', None, 'Logging in', | |
cb1553e9 | 46 | data=urllib.parse.urlencode({ |
7c74a015 JH |
47 | 'account': username, |
48 | 'password': password, | |
49 | 'session_id': session_id | |
50 | }).encode('ascii')) | |
51 | if login_response['code'] != 'ok': | |
97bef011 | 52 | raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True) |
44699d10 | 53 | if not self.is_logged_in: |
7c74a015 | 54 | raise ExtractorError('Login succeeded but did not set etp_rt cookie') |
80f48920 | 55 | |
cb1553e9 | 56 | def _get_embedded_json(self, webpage, display_id): |
f4d706a9 JH |
57 | initial_state = self._parse_json(self._search_regex( |
58 | r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'initial state'), display_id) | |
59 | app_config = self._parse_json(self._search_regex( | |
60 | r'__APP_CONFIG__\s*=\s*({.+?})\s*;', webpage, 'app config'), display_id) | |
61 | return initial_state, app_config | |
62 | ||
f4d706a9 | 63 | def _get_params(self, lang): |
cb1553e9 JH |
64 | if not CrunchyrollBaseIE.params: |
65 | if self._get_cookies(f'https://www.crunchyroll.com/{lang}').get('etp_rt'): | |
459262ac JH |
66 | grant_type, key = 'etp_rt_cookie', 'accountAuthClientId' |
67 | else: | |
68 | grant_type, key = 'client_id', 'anonClientId' | |
69 | ||
cb1553e9 JH |
70 | initial_state, app_config = self._get_embedded_json(self._download_webpage( |
71 | f'https://www.crunchyroll.com/{lang}', None, note='Retrieving main page'), None) | |
72 | api_domain = app_config['cxApiParams']['apiDomain'].replace('beta.crunchyroll.com', 'www.crunchyroll.com') | |
459262ac | 73 | |
f4d706a9 | 74 | auth_response = self._download_json( |
459262ac | 75 | f'{api_domain}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}', |
f4d706a9 | 76 | headers={ |
459262ac JH |
77 | 'Authorization': 'Basic ' + str(base64.b64encode(('%s:' % app_config['cxApiParams'][key]).encode('ascii')), 'ascii') |
78 | }, data=f'grant_type={grant_type}'.encode('ascii')) | |
f4d706a9 JH |
79 | policy_response = self._download_json( |
80 | f'{api_domain}/index/v2', None, note='Retrieving signed policy', | |
81 | headers={ | |
82 | 'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token'] | |
83 | }) | |
cb1553e9 | 84 | cms = policy_response.get('cms_web') |
88d62206 | 85 | bucket = cms['bucket'] |
f4d706a9 | 86 | params = { |
88d62206 JH |
87 | 'Policy': cms['policy'], |
88 | 'Signature': cms['signature'], | |
89 | 'Key-Pair-Id': cms['key_pair_id'] | |
f4d706a9 JH |
90 | } |
91 | locale = traverse_obj(initial_state, ('localization', 'locale')) | |
92 | if locale: | |
93 | params['locale'] = locale | |
cb1553e9 JH |
94 | CrunchyrollBaseIE.params = (api_domain, bucket, params) |
95 | return CrunchyrollBaseIE.params | |
f4d706a9 | 96 | |
f4d706a9 | 97 | |
cb1553e9 JH |
98 | class CrunchyrollBetaIE(CrunchyrollBaseIE): |
99 | IE_NAME = 'crunchyroll' | |
5da42f2b | 100 | _VALID_URL = r'''(?x) |
cb1553e9 | 101 | https?://(?:beta|www)\.crunchyroll\.com/ |
5da42f2b | 102 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) |
103 | watch/(?P<id>\w+) | |
104 | (?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)''' | |
dd078970 | 105 | _TESTS = [{ |
cb1553e9 | 106 | 'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future', |
dd078970 | 107 | 'info_dict': { |
b99ba3df | 108 | 'id': 'GY2P1Q98Y', |
dd078970 | 109 | 'ext': 'mp4', |
b99ba3df JH |
110 | 'duration': 1380.241, |
111 | 'timestamp': 1459632600, | |
dd078970 | 112 | 'description': 'md5:a022fbec4fbb023d43631032c91ed64b', |
dd078970 | 113 | 'title': 'World Trigger Episode 73 – To the Future', |
114 | 'upload_date': '20160402', | |
f4d706a9 | 115 | 'series': 'World Trigger', |
b99ba3df | 116 | 'series_id': 'GR757DMKY', |
f4d706a9 | 117 | 'season': 'World Trigger', |
b99ba3df | 118 | 'season_id': 'GR9P39NJ6', |
f4d706a9 | 119 | 'season_number': 1, |
b99ba3df JH |
120 | 'episode': 'To the Future', |
121 | 'episode_number': 73, | |
cb1553e9 | 122 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$', |
93abb740 | 123 | 'chapters': 'count:2', |
dd078970 | 124 | }, |
dfea94f8 SS |
125 | 'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'}, |
126 | }, { | |
cb1553e9 | 127 | 'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR', |
dfea94f8 SS |
128 | 'info_dict': { |
129 | 'id': 'GYE5WKQGR', | |
130 | 'ext': 'mp4', | |
131 | 'duration': 366.459, | |
132 | 'timestamp': 1476788400, | |
133 | 'description': 'md5:74b67283ffddd75f6e224ca7dc031e76', | |
134 | 'title': 'SHELTER Episode – Porter Robinson presents Shelter the Animation', | |
135 | 'upload_date': '20161018', | |
136 | 'series': 'SHELTER', | |
137 | 'series_id': 'GYGG09WWY', | |
138 | 'season': 'SHELTER', | |
139 | 'season_id': 'GR09MGK4R', | |
140 | 'season_number': 1, | |
141 | 'episode': 'Porter Robinson presents Shelter the Animation', | |
142 | 'episode_number': 0, | |
cb1553e9 | 143 | 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$', |
93abb740 | 144 | 'chapters': 'count:0', |
dfea94f8 SS |
145 | }, |
146 | 'params': {'skip_download': True}, | |
147 | 'skip': 'Video is Premium only', | |
f4d706a9 | 148 | }, { |
cb1553e9 | 149 | 'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y', |
f4d706a9 | 150 | 'only_matching': True, |
964b5493 | 151 | }, { |
152 | 'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy', | |
153 | 'only_matching': True, | |
dd078970 | 154 | }] |
155 | ||
156 | def _real_extract(self, url): | |
f4d706a9 | 157 | lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id') |
f4d706a9 JH |
158 | api_domain, bucket, params = self._get_params(lang) |
159 | ||
706dfe44 JH |
160 | episode_response = self._download_json( |
161 | f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id, | |
7d0f6f0c | 162 | note='Retrieving episode metadata', query=params) |
706dfe44 | 163 | if episode_response.get('is_premium_only') and not episode_response.get('playback'): |
44699d10 | 164 | if self.is_logged_in: |
165 | raise ExtractorError('This video is for premium members only', expected=True) | |
166 | else: | |
167 | self.raise_login_required('This video is for premium members only') | |
706dfe44 | 168 | |
f62f553d JH |
169 | stream_response = self._download_json( |
170 | f'{api_domain}{episode_response["__links__"]["streams"]["href"]}', display_id, | |
171 | note='Retrieving stream info', query=params) | |
7d0f6f0c | 172 | get_streams = lambda name: (traverse_obj(stream_response, name) or {}).items() |
706dfe44 JH |
173 | |
174 | requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])] | |
175 | hardsub_preference = qualities(requested_hardsubs[::-1]) | |
176 | requested_formats = self._configuration_arg('format') or ['adaptive_hls'] | |
177 | ||
dfea94f8 | 178 | available_formats = {} |
7d0f6f0c | 179 | for stream_type, streams in get_streams('streams'): |
706dfe44 JH |
180 | if stream_type not in requested_formats: |
181 | continue | |
182 | for stream in streams.values(): | |
706dfe44 JH |
183 | if not stream.get('url'): |
184 | continue | |
dfea94f8 SS |
185 | hardsub_lang = stream.get('hardsub_locale') or '' |
186 | format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s')) | |
187 | available_formats[hardsub_lang] = (stream_type, format_id, hardsub_lang, stream['url']) | |
188 | ||
189 | if '' in available_formats and 'all' not in requested_hardsubs: | |
190 | full_format_langs = set(requested_hardsubs) | |
191 | self.to_screen( | |
192 | 'To get all formats of a hardsub language, use ' | |
193 | '"--extractor-args crunchyrollbeta:hardsub=<language_code or all>". ' | |
08e29b9f | 194 | 'See https://github.com/yt-dlp/yt-dlp#crunchyrollbeta-crunchyroll for more info', |
dfea94f8 SS |
195 | only_once=True) |
196 | else: | |
197 | full_format_langs = set(map(str.lower, available_formats)) | |
198 | ||
199 | formats = [] | |
200 | for stream_type, format_id, hardsub_lang, stream_url in available_formats.values(): | |
201 | if stream_type.endswith('hls'): | |
202 | if hardsub_lang.lower() in full_format_langs: | |
706dfe44 | 203 | adaptive_formats = self._extract_m3u8_formats( |
dfea94f8 | 204 | stream_url, display_id, 'mp4', m3u8_id=format_id, |
7d0f6f0c | 205 | fatal=False, note=f'Downloading {format_id} HLS manifest') |
dfea94f8 SS |
206 | else: |
207 | adaptive_formats = (self._m3u8_meta_format(stream_url, ext='mp4', m3u8_id=format_id),) | |
208 | elif stream_type.endswith('dash'): | |
209 | adaptive_formats = self._extract_mpd_formats( | |
210 | stream_url, display_id, mpd_id=format_id, | |
211 | fatal=False, note=f'Downloading {format_id} MPD manifest') | |
212 | else: | |
213 | self.report_warning(f'Encountered unknown stream_type: {stream_type!r}', display_id, only_once=True) | |
214 | continue | |
215 | for f in adaptive_formats: | |
216 | if f.get('acodec') != 'none': | |
217 | f['language'] = stream_response.get('audio_locale') | |
218 | f['quality'] = hardsub_preference(hardsub_lang.lower()) | |
219 | formats.extend(adaptive_formats) | |
706dfe44 | 220 | |
93abb740 B |
221 | chapters = None |
222 | # if no intro chapter is available, a 403 without usable data is returned | |
223 | intro_chapter = self._download_json(f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json', | |
224 | display_id, fatal=False, errnote=False) | |
225 | if isinstance(intro_chapter, dict): | |
226 | chapters = [{ | |
227 | 'title': 'Intro', | |
228 | 'start_time': float_or_none(intro_chapter.get('startTime')), | |
229 | 'end_time': float_or_none(intro_chapter.get('endTime')) | |
230 | }] | |
231 | ||
706dfe44 JH |
232 | return { |
233 | 'id': internal_id, | |
7d0f6f0c B |
234 | 'title': '%s Episode %s – %s' % ( |
235 | episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')), | |
236 | 'description': try_get(episode_response, lambda x: x['description'].replace(r'\r\n', '\n')), | |
706dfe44 | 237 | 'duration': float_or_none(episode_response.get('duration_ms'), 1000), |
b99ba3df | 238 | 'timestamp': parse_iso8601(episode_response.get('upload_date')), |
706dfe44 JH |
239 | 'series': episode_response.get('series_title'), |
240 | 'series_id': episode_response.get('series_id'), | |
241 | 'season': episode_response.get('season_title'), | |
242 | 'season_id': episode_response.get('season_id'), | |
243 | 'season_number': episode_response.get('season_number'), | |
244 | 'episode': episode_response.get('title'), | |
245 | 'episode_number': episode_response.get('sequence_number'), | |
7d0f6f0c B |
246 | 'formats': formats, |
247 | 'thumbnails': [{ | |
248 | 'url': thumb.get('source'), | |
249 | 'width': thumb.get('width'), | |
250 | 'height': thumb.get('height'), | |
251 | } for thumb in traverse_obj(episode_response, ('images', 'thumbnail', ..., ...)) or []], | |
252 | 'subtitles': { | |
253 | lang: [{ | |
254 | 'url': subtitle_data.get('url'), | |
255 | 'ext': subtitle_data.get('format') | |
256 | }] for lang, subtitle_data in get_streams('subtitles') | |
257 | }, | |
93abb740 | 258 | 'chapters': chapters |
706dfe44 | 259 | } |
dd078970 | 260 | |
261 | ||
cb1553e9 JH |
262 | class CrunchyrollBetaShowIE(CrunchyrollBaseIE): |
263 | IE_NAME = 'crunchyroll:playlist' | |
5da42f2b | 264 | _VALID_URL = r'''(?x) |
cb1553e9 | 265 | https?://(?:beta|www)\.crunchyroll\.com/ |
5da42f2b | 266 | (?P<lang>(?:\w{2}(?:-\w{2})?/)?) |
267 | series/(?P<id>\w+) | |
268 | (?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)''' | |
dd078970 | 269 | _TESTS = [{ |
cb1553e9 | 270 | 'url': 'https://www.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA', |
dd078970 | 271 | 'info_dict': { |
b99ba3df | 272 | 'id': 'GY19NQ2QR', |
dd078970 | 273 | 'title': 'Girl Friend BETA', |
274 | }, | |
275 | 'playlist_mincount': 10, | |
276 | }, { | |
5da42f2b | 277 | 'url': 'https://beta.crunchyroll.com/it/series/GY19NQ2QR', |
dd078970 | 278 | 'only_matching': True, |
279 | }] | |
280 | ||
281 | def _real_extract(self, url): | |
f4d706a9 | 282 | lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id') |
f4d706a9 JH |
283 | api_domain, bucket, params = self._get_params(lang) |
284 | ||
285 | series_response = self._download_json( | |
286 | f'{api_domain}/cms/v2{bucket}/series/{internal_id}', display_id, | |
287 | note='Retrieving series metadata', query=params) | |
288 | ||
289 | seasons_response = self._download_json( | |
290 | f'{api_domain}/cms/v2{bucket}/seasons?series_id={internal_id}', display_id, | |
291 | note='Retrieving season list', query=params) | |
292 | ||
293 | def entries(): | |
294 | for season in seasons_response['items']: | |
295 | episodes_response = self._download_json( | |
296 | f'{api_domain}/cms/v2{bucket}/episodes?season_id={season["id"]}', display_id, | |
297 | note=f'Retrieving episode list for {season.get("slug_title")}', query=params) | |
298 | for episode in episodes_response['items']: | |
299 | episode_id = episode['id'] | |
300 | episode_display_id = episode['slug_title'] | |
301 | yield { | |
302 | '_type': 'url', | |
cb1553e9 | 303 | 'url': f'https://www.crunchyroll.com/{lang}watch/{episode_id}/{episode_display_id}', |
f4d706a9 JH |
304 | 'ie_key': CrunchyrollBetaIE.ie_key(), |
305 | 'id': episode_id, | |
306 | 'title': '%s Episode %s – %s' % (episode.get('season_title'), episode.get('episode'), episode.get('title')), | |
307 | 'description': try_get(episode, lambda x: x['description'].replace(r'\r\n', '\n')), | |
308 | 'duration': float_or_none(episode.get('duration_ms'), 1000), | |
309 | 'series': episode.get('series_title'), | |
310 | 'series_id': episode.get('series_id'), | |
311 | 'season': episode.get('season_title'), | |
312 | 'season_id': episode.get('season_id'), | |
313 | 'season_number': episode.get('season_number'), | |
314 | 'episode': episode.get('title'), | |
8a6b1677 CF |
315 | 'episode_number': episode.get('sequence_number'), |
316 | 'language': episode.get('audio_locale'), | |
f4d706a9 JH |
317 | } |
318 | ||
319 | return self.playlist_result(entries(), internal_id, series_response.get('title')) |