5 import xml
.etree
.ElementTree
7 from hashlib
import sha1
8 from math
import floor
, pow, sqrt
10 from .common
import InfoExtractor
11 from .vrv
import VRVBaseIE
12 from ..aes
import aes_cbc_decrypt
13 from ..compat
import (
15 compat_etree_fromstring
,
17 compat_urllib_parse_urlencode
,
40 class CrunchyrollBaseIE(InfoExtractor
):
41 _LOGIN_URL
= 'https://www.crunchyroll.com/welcome/login'
42 _API_BASE
= 'https://api.crunchyroll.com'
43 _NETRC_MACHINE
= 'crunchyroll'
45 def _call_rpc_api(self
, method
, video_id
, note
=None, data
=None):
47 data
['req'] = 'RpcApi' + method
48 data
= compat_urllib_parse_urlencode(data
).encode('utf-8')
49 return self
._download
_xml
(
50 'https://www.crunchyroll.com/xml/',
51 video_id
, note
, fatal
=False, data
=data
, headers
={
52 'Content-Type': 'application/x-www-form-urlencoded',
55 def _perform_login(self
, username
, password
):
56 if self
._get
_cookies
(self
._LOGIN
_URL
).get('etp_rt'):
59 upsell_response
= self
._download
_json
(
60 f
'{self._API_BASE}/get_upsell_data.0.json', None, 'Getting session id',
63 'device_id': 'whatvalueshouldbeforweb',
64 'device_type': 'com.crunchyroll.static',
65 'access_token': 'giKq5eY27ny3cqz',
66 'referer': self
._LOGIN
_URL
68 if upsell_response
['code'] != 'ok':
69 raise ExtractorError('Could not get session id')
70 session_id
= upsell_response
['data']['session_id']
72 login_response
= self
._download
_json
(
73 f
'{self._API_BASE}/login.1.json', None, 'Logging in',
74 data
=compat_urllib_parse_urlencode({
77 'session_id': session_id
79 if login_response
['code'] != 'ok':
80 raise ExtractorError('Login failed. Server message: %s' % login_response
['message'], expected
=True)
81 if not self
._get
_cookies
(self
._LOGIN
_URL
).get('etp_rt'):
82 raise ExtractorError('Login succeeded but did not set etp_rt cookie')
84 # Beta-specific, but needed for redirects
85 def _get_beta_embedded_json(self
, webpage
, display_id
):
86 initial_state
= self
._parse
_json
(self
._search
_regex
(
87 r
'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage
, 'initial state'), display_id
)
88 app_config
= self
._parse
_json
(self
._search
_regex
(
89 r
'__APP_CONFIG__\s*=\s*({.+?})\s*;', webpage
, 'app config'), display_id
)
90 return initial_state
, app_config
92 def _redirect_to_beta(self
, webpage
, iekey
, video_id
):
93 if not self
._get
_cookies
(self
._LOGIN
_URL
).get('etp_rt'):
94 raise ExtractorError('Received a beta page from non-beta url when not logged in.')
95 initial_state
, app_config
= self
._get
_beta
_embedded
_json
(webpage
, video_id
)
96 url
= app_config
['baseSiteUrl'] + initial_state
['router']['locations']['current']['pathname']
97 self
.to_screen(f
'{video_id}: Redirected to beta site - {url}')
98 return self
.url_result(f
'{url}', iekey
, video_id
)
101 def _add_skip_wall(url
):
102 parsed_url
= compat_urlparse
.urlparse(url
)
103 qs
= compat_urlparse
.parse_qs(parsed_url
.query
)
104 # Always force skip_wall to bypass maturity wall, namely 18+ confirmation message:
105 # > This content may be inappropriate for some people.
106 # > Are you sure you want to continue?
107 # since it's not disabled by default in crunchyroll account's settings.
108 # See https://github.com/ytdl-org/youtube-dl/issues/7202.
109 qs
['skip_wall'] = ['1']
110 return compat_urlparse
.urlunparse(
111 parsed_url
._replace
(query
=compat_urllib_parse_urlencode(qs
, True)))
114 class CrunchyrollIE(CrunchyrollBaseIE
, VRVBaseIE
):
115 IE_NAME
= 'crunchyroll'
116 _VALID_URL
= r
'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|(?!series/|watch/)(?:[^/]+/){1,2}[^/?&]*?)(?P<id>[0-9]+))(?:[/?&]|$)'
118 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
122 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
123 'description': 'md5:2d17137920c64f2f49981a7797d275ef',
124 'thumbnail': r
're:^https?://.*\.jpg$',
125 'uploader': 'Yomiuri Telecasting Corporation (YTV)',
126 'upload_date': '20131013',
127 'url': 're:(?!.*&)',
131 'skip_download': True,
133 'skip': 'Video gone',
135 'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1',
139 'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
140 'description': 'md5:2fbc01f90b87e8e9137296f37b461c12',
141 'thumbnail': r
're:^https?://.*\.jpg$',
142 'uploader': 'Danny Choo Network',
143 'upload_date': '20120213',
147 'skip_download': True,
149 'skip': 'Video gone',
151 'url': 'http://www.crunchyroll.com/rezero-starting-life-in-another-world-/episode-5-the-morning-of-our-promise-is-still-distant-702409',
156 'description': compat_str
,
157 'thumbnail': r
're:^https?://.*\.jpg$',
158 'uploader': 'Re:Zero Partners',
159 'timestamp': 1462098900,
160 'upload_date': '20160501',
164 'skip_download': True,
167 'url': 'http://www.crunchyroll.com/konosuba-gods-blessing-on-this-wonderful-world/episode-1-give-me-deliverance-from-this-judicial-injustice-727589',
172 'description': compat_str
,
173 'thumbnail': r
're:^https?://.*\.jpg$',
174 'uploader': 'Kadokawa Pictures Inc.',
175 'timestamp': 1484130900,
176 'upload_date': '20170111',
177 'series': compat_str
,
178 'season': "KONOSUBA -God's blessing on this wonderful world! 2",
180 'episode': 'Give Me Deliverance From This Judicial Injustice!',
185 'skip_download': True,
188 'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
189 'only_matching': True,
191 # geo-restricted (US), 18+ maturity wall, non-premium available
192 'url': 'http://www.crunchyroll.com/cosplay-complex-ova/episode-1-the-birth-of-the-cosplay-club-565617',
193 'only_matching': True,
195 # A description with double quotes
196 'url': 'http://www.crunchyroll.com/11eyes/episode-1-piros-jszaka-red-night-535080',
201 'description': compat_str
,
202 'uploader': 'Marvelous AQL Inc.',
203 'timestamp': 1255512600,
204 'upload_date': '20091014',
207 # Just test metadata extraction
208 'skip_download': True,
211 # make sure we can extract an uploader name that's not a link
212 'url': 'http://www.crunchyroll.com/hakuoki-reimeiroku/episode-1-dawn-of-the-divine-warriors-606899',
216 'title': 'Hakuoki Reimeiroku Episode 1 – Dawn of the Divine Warriors',
217 'description': 'Ryunosuke was left to die, but Serizawa-san asked him a simple question "Do you want to live?"',
218 'uploader': 'Geneon Entertainment',
219 'upload_date': '20120717',
222 # just test metadata extraction
223 'skip_download': True,
225 'skip': 'Video gone',
227 # A video with a vastly different season name compared to the series name
228 'url': 'http://www.crunchyroll.com/nyarko-san-another-crawling-chaos/episode-1-test-590532',
233 'description': compat_str
,
234 'uploader': 'TV TOKYO',
235 'timestamp': 1330956000,
236 'upload_date': '20120305',
237 'series': 'Nyarko-san: Another Crawling Chaos',
238 'season': 'Haiyoru! Nyaruani (ONA)',
241 # Just test metadata extraction
242 'skip_download': True,
245 'url': 'http://www.crunchyroll.com/media-723735',
246 'only_matching': True,
248 'url': 'https://www.crunchyroll.com/en-gb/mob-psycho-100/episode-2-urban-legends-encountering-rumors-780921',
249 'only_matching': True,
253 '360': ('60', '106'),
254 '480': ('61', '106'),
255 '720': ('62', '106'),
256 '1080': ('80', '108'),
259 def _download_webpage(self
, url_or_request
, *args
, **kwargs
):
260 request
= (url_or_request
if isinstance(url_or_request
, urllib
.request
.Request
)
261 else sanitized_Request(url_or_request
))
262 # Accept-Language must be set explicitly to accept any language to avoid issues
263 # similar to https://github.com/ytdl-org/youtube-dl/issues/6797.
264 # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
265 # should be imposed or not (from what I can see it just takes the first language
266 # ignoring the priority and requires it to correspond the IP). By the way this causes
267 # Crunchyroll to not work in georestriction cases in some browsers that don't place
268 # the locale lang first in header. However allowing any language seems to workaround the issue.
269 request
.add_header('Accept-Language', '*')
270 return super(CrunchyrollBaseIE
, self
)._download
_webpage
(request
, *args
, **kwargs
)
272 def _decrypt_subtitles(self
, data
, iv
, id):
273 data
= bytes_to_intlist(compat_b64decode(data
))
274 iv
= bytes_to_intlist(compat_b64decode(iv
))
277 def obfuscate_key_aux(count
, modulo
, start
):
279 for _
in range(count
):
280 output
.append(output
[-1] + output
[-2])
281 # cut off start values
283 output
= list(map(lambda x
: x
% modulo
+ 33, output
))
286 def obfuscate_key(key
):
287 num1
= int(floor(pow(2, 25) * sqrt(6.9)))
288 num2
= (num1 ^ key
) << 5
290 num4
= num3 ^
(num3
>> 3) ^ num2
291 prefix
= intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2)))
292 shaHash
= bytes_to_intlist(sha1(prefix
+ str(num4
).encode('ascii')).digest())
293 # Extend 160 Bit hash to 256 Bit
294 return shaHash
+ [0] * 12
296 key
= obfuscate_key(id)
298 decrypted_data
= intlist_to_bytes(aes_cbc_decrypt(data
, key
, iv
))
299 return zlib
.decompress(decrypted_data
)
301 def _convert_subtitles_to_srt(self
, sub_root
):
304 for i
, event
in enumerate(sub_root
.findall('./events/event'), 1):
305 start
= event
.attrib
['start'].replace('.', ',')
306 end
= event
.attrib
['end'].replace('.', ',')
307 text
= event
.attrib
['text'].replace('\\N', '\n')
308 output
+= '%d\n%s --> %s\n%s\n\n' % (i
, start
, end
, text
)
311 def _convert_subtitles_to_ass(self
, sub_root
):
314 def ass_bool(strvalue
):
320 output
= '[Script Info]\n'
321 output
+= 'Title: %s\n' % sub_root
.attrib
['title']
322 output
+= 'ScriptType: v4.00+\n'
323 output
+= 'WrapStyle: %s\n' % sub_root
.attrib
['wrap_style']
324 output
+= 'PlayResX: %s\n' % sub_root
.attrib
['play_res_x']
325 output
+= 'PlayResY: %s\n' % sub_root
.attrib
['play_res_y']
328 Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
330 for style
in sub_root
.findall('./styles/style'):
331 output
+= 'Style: ' + style
.attrib
['name']
332 output
+= ',' + style
.attrib
['font_name']
333 output
+= ',' + style
.attrib
['font_size']
334 output
+= ',' + style
.attrib
['primary_colour']
335 output
+= ',' + style
.attrib
['secondary_colour']
336 output
+= ',' + style
.attrib
['outline_colour']
337 output
+= ',' + style
.attrib
['back_colour']
338 output
+= ',' + ass_bool(style
.attrib
['bold'])
339 output
+= ',' + ass_bool(style
.attrib
['italic'])
340 output
+= ',' + ass_bool(style
.attrib
['underline'])
341 output
+= ',' + ass_bool(style
.attrib
['strikeout'])
342 output
+= ',' + style
.attrib
['scale_x']
343 output
+= ',' + style
.attrib
['scale_y']
344 output
+= ',' + style
.attrib
['spacing']
345 output
+= ',' + style
.attrib
['angle']
346 output
+= ',' + style
.attrib
['border_style']
347 output
+= ',' + style
.attrib
['outline']
348 output
+= ',' + style
.attrib
['shadow']
349 output
+= ',' + style
.attrib
['alignment']
350 output
+= ',' + style
.attrib
['margin_l']
351 output
+= ',' + style
.attrib
['margin_r']
352 output
+= ',' + style
.attrib
['margin_v']
353 output
+= ',' + style
.attrib
['encoding']
358 Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
360 for event
in sub_root
.findall('./events/event'):
361 output
+= 'Dialogue: 0'
362 output
+= ',' + event
.attrib
['start']
363 output
+= ',' + event
.attrib
['end']
364 output
+= ',' + event
.attrib
['style']
365 output
+= ',' + event
.attrib
['name']
366 output
+= ',' + event
.attrib
['margin_l']
367 output
+= ',' + event
.attrib
['margin_r']
368 output
+= ',' + event
.attrib
['margin_v']
369 output
+= ',' + event
.attrib
['effect']
370 output
+= ',' + event
.attrib
['text']
375 def _extract_subtitles(self
, subtitle
):
376 sub_root
= compat_etree_fromstring(subtitle
)
379 'data': self
._convert
_subtitles
_to
_srt
(sub_root
),
382 'data': self
._convert
_subtitles
_to
_ass
(sub_root
),
385 def _get_subtitles(self
, video_id
, webpage
):
387 for sub_id
, sub_name
in re
.findall(r
'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage
):
388 sub_doc
= self
._call
_rpc
_api
(
389 'Subtitle_GetXml', video_id
,
390 'Downloading subtitles for ' + sub_name
, data
={
391 'subtitle_script_id': sub_id
,
393 if not isinstance(sub_doc
, xml
.etree
.ElementTree
.Element
):
395 sid
= sub_doc
.get('id')
396 iv
= xpath_text(sub_doc
, 'iv', 'subtitle iv')
397 data
= xpath_text(sub_doc
, 'data', 'subtitle data')
398 if not sid
or not iv
or not data
:
400 subtitle
= self
._decrypt
_subtitles
(data
, iv
, sid
).decode('utf-8')
401 lang_code
= self
._search
_regex
(r
'lang_code=["\']([^
"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
404 subtitles[lang_code] = self._extract_subtitles(subtitle)
407 def _real_extract(self, url):
408 mobj = self._match_valid_url(url)
409 video_id = mobj.group('id')
411 if mobj.group('prefix') == 'm':
412 mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage')
413 webpage_url = self._search_regex(r'<link rel="canonical
" href="([^
"]+)" />', mobile_webpage, 'webpage_url
')
415 webpage_url = 'http
://www
.' + mobj.group('url
')
417 webpage = self._download_webpage(
418 self._add_skip_wall(webpage_url), video_id,
419 headers=self.geo_verification_headers())
420 if re.search(r'<div
id="preload-data">', webpage):
421 return self._redirect_to_beta(webpage, CrunchyrollBetaIE.ie_key(), video_id)
422 note_m = self._html_search_regex(
423 r'<div
class="showmedia-trailer-notice">(.+?
)</div
>',
424 webpage, 'trailer
-notice
', default='')
426 raise ExtractorError(note_m, expected=True)
428 mobj = re.search(r'Page\
.messaging_box_controller\
.addItems\
(\
[(?P
<msg
>{.+?}
)\
]\
)', webpage)
430 msg = json.loads(mobj.group('msg
'))
431 if msg.get('type') == 'error
':
432 raise ExtractorError('crunchyroll returned error
: %s' % msg['message_body
'], expected=True)
434 if 'To view this
, please log
in to verify you are
18 or older
.' in webpage:
435 self.raise_login_required()
437 media = self._parse_json(self._search_regex(
438 r'vilos\
.config\
.media\s
*=\s
*({.+?}
);',
439 webpage, 'vilos media
', default='{}'), video_id)
440 media_metadata = media.get('metadata
') or {}
442 language = self._search_regex(
443 r'(?
:vilos\
.config\
.player\
.language|LOCALE
)\s
*=\s
*(["\'])(?P<lang>(?:(?!\1).)+)\1',
444 webpage, 'language', default=None, group='lang')
446 video_title = self._html_search_regex(
447 (r'(?s)<h1[^>]*>((?:(?!<h1).)*?<(?:span[^>]+itemprop=["\']title
["\']|meta[^>]+itemprop=["\']position
["\'])[^>]*>(?:(?!<h1).)+?)</h1>',
448 r'<title>(.+?),\s+-\s+.+? Crunchyroll'),
449 webpage, 'video_title', default=None)
451 video_title = re.sub(r'^Watch\s+', '', self._og_search_description(webpage))
452 video_title = re.sub(r' {2,}', ' ', video_title)
453 video_description = (self._parse_json(self._html_search_regex(
454 r'<script[^>]*>\s*.+?\[media_id=%s\].+?({.+?"description"\s*:.+?})\);' % video_id,
455 webpage, 'description', default='{}'), video_id) or media_metadata).get('description')
458 thumbnail_url = (self._parse_json(self._html_search_regex(
459 r'<script type="application\
/ld\
+json
">\n\s*(.+?)<\/script>',
460 webpage, 'thumbnail_url', default='{}'), video_id)).get('image')
463 'url': thumbnail_url,
468 if video_description:
469 video_description = lowercase_escape(video_description.replace(r'\r\n', '\n'))
470 video_uploader = self._html_search_regex(
471 # try looking for both an uploader that's a link and one that's not
472 [r'<a[^>]+href="/publisher
/[^
"]+"[^
>]*>([^
<]+)</a
>', r'<div
>\s
*Publisher
:\s
*<span
>\s
*(.+?
)\s
*</span
>\s
*</div
>'],
473 webpage, 'video_uploader
', default=False)
475 requested_languages = self._configuration_arg('language
')
476 requested_hardsubs = [('' if val == 'none
' else val) for val in self._configuration_arg('hardsub
')]
477 language_preference = qualities((requested_languages or [language or ''])[::-1])
478 hardsub_preference = qualities((requested_hardsubs or ['', language or ''])[::-1])
481 for stream in media.get('streams
', []):
482 audio_lang = stream.get('audio_lang
') or ''
483 hardsub_lang = stream.get('hardsub_lang
') or ''
484 if (requested_languages and audio_lang.lower() not in requested_languages
485 or requested_hardsubs and hardsub_lang.lower() not in requested_hardsubs):
487 vrv_formats = self._extract_vrv_formats(
488 stream.get('url
'), video_id, stream.get('format
'),
489 audio_lang, hardsub_lang)
490 for f in vrv_formats:
491 f['language_preference
'] = language_preference(audio_lang)
492 f['quality
'] = hardsub_preference(hardsub_lang)
493 formats.extend(vrv_formats)
496 for a, fmt in re.findall(r'(<a
[^
>]+token
=["\']showmedia\.([0-9]{3,4})p["\'][^
>]+>)', webpage):
497 attrs = extract_attributes(a)
498 href = attrs.get('href
')
499 if href and '/freetrial
' in href:
501 available_fmts.append(fmt)
502 if not available_fmts:
503 for p in (r'token
=["\']showmedia\.([0-9]{3,4})p"', r'showmedia\
.([0-9]{3,4}
)p
'):
504 available_fmts = re.findall(p, webpage)
507 if not available_fmts:
508 available_fmts = self._FORMAT_IDS.keys()
509 video_encode_ids = []
511 for fmt in available_fmts:
512 stream_quality, stream_format = self._FORMAT_IDS[fmt]
513 video_format = fmt + 'p
'
515 streamdata = self._call_rpc_api(
516 'VideoPlayer_GetStandardConfig
', video_id,
517 'Downloading media info
for %s' % video_format, data={
518 'media_id
': video_id,
519 'video_format
': stream_format,
520 'video_quality
': stream_quality,
523 if isinstance(streamdata, xml.etree.ElementTree.Element):
524 stream_info = streamdata.find('./{default}preload
/stream_info
')
525 if stream_info is not None:
526 stream_infos.append(stream_info)
527 stream_info = self._call_rpc_api(
528 'VideoEncode_GetStreamInfo
', video_id,
529 'Downloading stream info
for %s' % video_format, data={
530 'media_id
': video_id,
531 'video_format
': stream_format,
532 'video_encode_quality
': stream_quality,
534 if isinstance(stream_info, xml.etree.ElementTree.Element):
535 stream_infos.append(stream_info)
536 for stream_info in stream_infos:
537 video_encode_id = xpath_text(stream_info, './video_encode_id
')
538 if video_encode_id in video_encode_ids:
540 video_encode_ids.append(video_encode_id)
542 video_file = xpath_text(stream_info, './file')
545 if video_file.startswith('http
'):
546 formats.extend(self._extract_m3u8_formats(
547 video_file, video_id, 'mp4
', entry_protocol='m3u8_native
',
548 m3u8_id='hls
', fatal=False))
551 video_url = xpath_text(stream_info, './host
')
554 metadata = stream_info.find('./metadata
')
556 'format
': video_format,
557 'height
': int_or_none(xpath_text(metadata, './height
')),
558 'width
': int_or_none(xpath_text(metadata, './width
')),
561 if '.fplive
.net
/' in video_url:
562 video_url = re.sub(r'^rtmpe?
://', 'http
://', video_url.strip())
563 parsed_video_url = compat_urlparse.urlparse(video_url)
564 direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
565 netloc='v
.lvlt
.crcdn
.net
',
566 path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
567 if self._is_valid_url(direct_video_url, video_id, video_format):
569 'format_id
': 'http
-' + video_format,
570 'url
': direct_video_url,
572 formats.append(format_info)
576 'format_id
': 'rtmp
-' + video_format,
578 'play_path
': video_file,
581 formats.append(format_info)
582 self._sort_formats(formats)
584 metadata = self._call_rpc_api(
585 'VideoPlayer_GetMediaMetadata
', video_id,
586 note='Downloading media info
', data={
587 'media_id
': video_id,
591 for subtitle in media.get('subtitles
', []):
592 subtitle_url = subtitle.get('url
')
595 subtitles.setdefault(subtitle.get('language
', 'enUS
'), []).append({
597 'ext
': subtitle.get('format
', 'ass
'),
600 subtitles = self.extract_subtitles(video_id, webpage)
602 # webpage provide more accurate data than series_title from XML
603 series = self._html_search_regex(
604 r'(?s
)<h\d
[^
>]+\bid
=["\']showmedia_about_episode_num[^>]+>(.+?)</h\d',
605 webpage, 'series', fatal=False)
607 season = episode = episode_number = duration = None
609 if isinstance(metadata, xml.etree.ElementTree.Element):
610 season = xpath_text(metadata, 'series_title')
611 episode = xpath_text(metadata, 'episode_title')
612 episode_number = int_or_none(xpath_text(metadata, 'episode_number'))
613 duration = float_or_none(media_metadata.get('duration'), 1000)
616 episode = media_metadata.get('title')
617 if not episode_number:
618 episode_number = int_or_none(media_metadata.get('episode_number'))
619 thumbnail_url = try_get(media, lambda x: x['thumbnail']['url'])
622 'url': thumbnail_url,
627 season_number = int_or_none(self._search_regex(
628 r'(?s)<h\d[^>]+id=["\']showmedia_about_episode_num
[^
>]+>.+?
</h\d
>\s
*<h4
>\s
*Season (\d
+)',
629 webpage, 'season number
', default=None))
631 info = self._search_json_ld(webpage, video_id, default={})
635 'title
': video_title,
636 'description
': video_description,
637 'duration
': duration,
638 'thumbnails
': thumbnails,
639 'uploader
': video_uploader,
642 'season_number
': season_number,
644 'episode_number
': episode_number,
645 'subtitles
': subtitles,
650 class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
651 IE_NAME = 'crunchyroll
:playlist
'
652 _VALID_URL = r'https?
://(?
:(?P
<prefix
>www|m
)\
.)?
(?P
<url
>crunchyroll\
.com
/(?
:\w{2}
(?
:-\w{2}
)?
/)?
(?
!(?
:news|anime
-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login|media
-\d
+))(?P
<id>[\w\
-]+))/?
(?
:\?|$
)'
655 'url
': 'https
://www
.crunchyroll
.com
/a
-bridge
-to
-the
-starry
-skies
-hoshizora
-e
-kakaru
-hashi
',
657 'id': 'a
-bridge
-to
-the
-starry
-skies
-hoshizora
-e
-kakaru
-hashi
',
658 'title
': 'A Bridge to the Starry Skies
- Hoshizora e Kakaru Hashi
'
660 'playlist_count
': 13,
662 # geo-restricted (US), 18+ maturity wall, non-premium available
663 'url
': 'http
://www
.crunchyroll
.com
/cosplay
-complex-ova
',
665 'id': 'cosplay
-complex-ova
',
666 'title
': 'Cosplay Complex OVA
'
669 'skip
': 'Georestricted
',
671 # geo-restricted (US), 18+ maturity wall, non-premium will be available since 2015.11.14
672 'url
': 'http
://www
.crunchyroll
.com
/ladies
-versus
-butlers?skip_wall
=1',
673 'only_matching
': True,
675 'url
': 'http
://www
.crunchyroll
.com
/fr
/ladies
-versus
-butlers
',
676 'only_matching
': True,
679 def _real_extract(self, url):
680 show_id = self._match_id(url)
682 webpage = self._download_webpage(
683 # https:// gives a 403, but http:// does not
684 self._add_skip_wall(url).replace('https
://', 'http
://'), show_id,
685 headers=self.geo_verification_headers())
686 if re.search(r'<div
id="preload-data">', webpage):
687 return self._redirect_to_beta(webpage, CrunchyrollBetaShowIE.ie_key(), show_id)
688 title = self._html_search_meta('name
', webpage, default=None)
690 episode_re = r'<li
id="showview_videos_media_(\d+)"[^
>]+>.*?
<a href
="([^"]+)"'
691 season_re = r'<a [^>]+season-dropdown[^>]+>([^<]+)'
692 paths = re.findall(f'(?s){episode_re}|{season_re}', webpage)
694 entries, current_season = [], None
695 for ep_id, ep, season in paths:
697 current_season = season
699 entries.append(self.url_result(
700 f'http://www.crunchyroll.com{ep}', CrunchyrollIE.ie_key(), ep_id, season=current_season))
706 'entries': reversed(entries),
710 class CrunchyrollBetaBaseIE(CrunchyrollBaseIE):
713 def _get_params(self, lang):
714 if not CrunchyrollBetaBaseIE.params:
715 initial_state, app_config = self._get_beta_embedded_json(self._download_webpage(
716 f'https://beta.crunchyroll.com/{lang}', None, note='Retrieving main page'), None)
717 api_domain = app_config['cxApiParams']['apiDomain']
718 basic_token = str(base64.b64encode(('%s:' % app_config['cxApiParams']['accountAuthClientId']).encode('ascii')), 'ascii')
719 auth_response = self._download_json(
720 f'{api_domain}/auth/v1/token', None, note='Authenticating with cookie',
722 'Authorization': 'Basic ' + basic_token
723 }, data='grant_type=etp_rt_cookie'.encode('ascii'))
724 policy_response = self._download_json(
725 f'{api_domain}/index/v2', None, note='Retrieving signed policy',
727 'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']
729 cms = traverse_obj(policy_response, 'cms_beta', 'cms')
730 bucket = cms['bucket']
732 'Policy': cms['policy'],
733 'Signature': cms['signature'],
734 'Key-Pair-Id': cms['key_pair_id']
736 locale = traverse_obj(initial_state, ('localization', 'locale'))
738 params['locale'] = locale
739 CrunchyrollBetaBaseIE.params = (api_domain, bucket, params)
740 return CrunchyrollBetaBaseIE.params
742 def _redirect_from_beta(self, url, lang, internal_id, display_id, is_episode, iekey):
743 initial_state, app_config = self._get_beta_embedded_json(self._download_webpage(url, display_id), display_id)
744 content_data = initial_state['content']['byId'][internal_id]
746 video_id = content_data['external_id'].split('.')[1]
747 series_id = content_data['episode_metadata']['series_slug_title']
749 series_id = content_data['slug_title']
750 series_id = re.sub(r'-{2,}', '-', series_id)
751 url = f'https://www.crunchyroll.com/{lang}{series_id}'
753 url = url + f'/{display_id}-{video_id}'
754 self.to_screen(f'{display_id}: Not logged in. Redirecting to non-beta site - {url}')
755 return self.url_result(url, iekey, display_id)
758 class CrunchyrollBetaIE(CrunchyrollBetaBaseIE):
759 IE_NAME = 'crunchyroll:beta'
760 _VALID_URL = r'https?://beta\.crunchyroll\.com/(?P<lang>(?:\w{2}(?:-\w{2})?/)?)watch/(?P<id>\w+)/(?P<display_id>[\w\-]*)/?(?:\?|$)'
762 'url': 'https://beta.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
766 'timestamp': 1459610100,
767 'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
768 'uploader': 'Toei Animation',
769 'title': 'World Trigger Episode 73 – To the Future',
770 'upload_date': '20160402',
771 'episode_number': 73,
772 'series': 'World Trigger',
773 'average_rating': 4.9,
774 'episode': 'To the Future',
775 'season': 'World Trigger',
776 'thumbnail': 'https://img1.ak.crunchyroll.com/i/spire3-tmb/c870dedca1a83137c2d3d144984155ed1459527119_main.jpg',
779 'params': {'skip_download': 'm3u8'},
780 'expected_warnings': ['Unable to download XML']
782 'url': 'https://beta.crunchyroll.com/watch/GYK53DMPR/wicked-lord-shingan-reborn',
787 'timestamp': 1389173400,
788 'series': 'Love, Chunibyo & Other Delusions - Heart Throb -',
789 'description': 'md5:5579d1a0355cc618558ba23d27067a62',
791 'episode': 'Wicked Lord Shingan... Reborn',
792 'average_rating': 4.9,
793 'season': 'Love, Chunibyo & Other Delusions - Heart Throb -',
794 'thumbnail': 'https://img1.ak.crunchyroll.com/i/spire3-tmb/2ba0384e225a5370d5f0ee9496d91ea51389046521_main.jpg',
795 'title': 'Love, Chunibyo & Other Delusions - Heart Throb - Episode 1 – Wicked Lord Shingan... Reborn',
797 'upload_date': '20140108',
799 'params': {'skip_download': 'm3u8'},
800 'expected_warnings': ['Unable to download XML']
802 'url': 'https://beta.crunchyroll.com/watch/GY2P1Q98Y/',
803 'only_matching': True,
806 def _real_extract(self, url):
807 lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
809 if not self._get_cookies(url).get('etp_rt'):
810 return self._redirect_from_beta(url, lang, internal_id, display_id, True, CrunchyrollIE.ie_key())
812 api_domain, bucket, params = self._get_params(lang)
814 episode_response = self._download_json(
815 f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
816 note='Retrieving episode metadata', query=params)
817 if episode_response.get('is_premium_only') and not episode_response.get('playback'):
818 raise ExtractorError('This video is for premium members only.', expected=True)
820 stream_response = self._download_json(episode_response['playback'], display_id, note='Retrieving stream info')
821 get_streams = lambda name: (traverse_obj(stream_response, name) or {}).items()
823 requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
824 hardsub_preference = qualities(requested_hardsubs[::-1])
825 requested_formats = self._configuration_arg('format') or ['adaptive_hls']
828 for stream_type, streams in get_streams('streams'):
829 if stream_type not in requested_formats:
831 for stream in streams.values():
832 hardsub_lang = stream.get('hardsub_locale') or ''
833 if hardsub_lang.lower() not in requested_hardsubs:
835 format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s'))
836 if not stream.get('url'):
838 if stream_type.endswith('hls'):
839 adaptive_formats = self._extract_m3u8_formats(
840 stream['url'], display_id, 'mp4', m3u8_id=format_id,
841 fatal=False, note=f'Downloading {format_id} HLS manifest')
842 elif stream_type.endswith('dash'):
843 adaptive_formats = self._extract_mpd_formats(
844 stream['url'], display_id, mpd_id=format_id,
845 fatal=False, note=f'Downloading {format_id} MPD manifest')
846 for f in adaptive_formats:
847 if f.get('acodec') != 'none':
848 f['language'] = stream_response.get('audio_locale')
849 f['quality'] = hardsub_preference(hardsub_lang.lower())
850 formats.extend(adaptive_formats)
851 self._sort_formats(formats)
855 'title': '%s Episode %s – %s' % (
856 episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
857 'description': try_get(episode_response, lambda x: x['description'].replace(r'\r\n', '\n')),
858 'duration': float_or_none(episode_response.get('duration_ms'), 1000),
859 'series': episode_response.get('series_title'),
860 'series_id': episode_response.get('series_id'),
861 'season': episode_response.get('season_title'),
862 'season_id': episode_response.get('season_id'),
863 'season_number': episode_response.get('season_number'),
864 'episode': episode_response.get('title'),
865 'episode_number': episode_response.get('sequence_number'),
868 'url': thumb.get('source'),
869 'width': thumb.get('width'),
870 'height': thumb.get('height'),
871 } for thumb in traverse_obj(episode_response, ('images', 'thumbnail', ..., ...)) or []],
874 'url': subtitle_data.get('url'),
875 'ext': subtitle_data.get('format')
876 }] for lang, subtitle_data in get_streams('subtitles')
881 class CrunchyrollBetaShowIE(CrunchyrollBetaBaseIE):
882 IE_NAME = 'crunchyroll:playlist:beta'
883 _VALID_URL = r'https?://beta\.crunchyroll\.com/(?P<lang>(?:\w{2}(?:-\w{2})?/)?)series/(?P<id>\w+)/(?P<display_id>[\w\-]*)/?(?:\?|$)'
885 'url': 'https://beta.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA',
887 'id': 'girl-friend-beta',
888 'title': 'Girl Friend BETA',
890 'playlist_mincount': 10,
892 'url': 'https://beta.crunchyroll.com/series/GYJQV73V6/love-chunibyo--other-delusions---heart-throb--',
894 'id': 'love-chunibyo-other-delusions-heart-throb-',
895 'title': 'Love, Chunibyo & Other Delusions - Heart Throb -',
897 'playlist_mincount': 10,
899 'url': 'https://beta.crunchyroll.com/it/series/GY19NQ2QR/Girl-Friend-BETA',
900 'only_matching': True,
902 'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
903 'only_matching': True,
906 def _real_extract(self, url):
907 lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
909 if not self._get_cookies(url).get('etp_rt'):
910 return self._redirect_from_beta(url, lang, internal_id, display_id, False, CrunchyrollShowPlaylistIE.ie_key())
912 api_domain, bucket, params = self._get_params(lang)
914 series_response = self._download_json(
915 f'{api_domain}/cms/v2{bucket}/series/{internal_id}', display_id,
916 note='Retrieving series metadata', query=params)
918 seasons_response = self._download_json(
919 f'{api_domain}/cms/v2{bucket}/seasons?series_id={internal_id}', display_id,
920 note='Retrieving season list', query=params)
923 for season in seasons_response['items']:
924 episodes_response = self._download_json(
925 f'{api_domain}/cms/v2{bucket}/episodes?season_id={season["id"]}', display_id,
926 note=f'Retrieving episode list for {season.get("slug_title")}', query=params)
927 for episode in episodes_response['items']:
928 episode_id = episode['id']
929 episode_display_id = episode['slug_title']
932 'url': f'https://beta.crunchyroll.com/{lang}watch/{episode_id}/{episode_display_id}',
933 'ie_key': CrunchyrollBetaIE.ie_key(),
935 'title': '%s Episode %s – %s' % (episode.get('season_title'), episode.get('episode'), episode.get('title')),
936 'description': try_get(episode, lambda x: x['description'].replace(r'\r\n', '\n')),
937 'duration': float_or_none(episode.get('duration_ms'), 1000),
938 'series': episode.get('series_title'),
939 'series_id': episode.get('series_id'),
940 'season': episode.get('season_title'),
941 'season_id': episode.get('season_id'),
942 'season_number': episode.get('season_number'),
943 'episode': episode.get('title'),
944 'episode_number': episode.get('sequence_number')
947 return self.playlist_result(entries(), internal_id, series_response.get('title'))