2 from __future__
import unicode_literals
9 from hashlib
import sha1
10 from math
import pow, sqrt
, floor
11 from .common
import InfoExtractor
12 from .vrv
import VRVBaseIE
13 from ..compat
import (
16 compat_etree_fromstring
,
18 compat_urllib_parse_urlencode
,
19 compat_urllib_request
,
45 class CrunchyrollBaseIE(InfoExtractor
):
46 _LOGIN_URL
= 'https://www.crunchyroll.com/welcome/login'
47 _API_BASE
= 'https://api.crunchyroll.com'
48 _NETRC_MACHINE
= 'crunchyroll'
50 def _call_rpc_api(self
, method
, video_id
, note
=None, data
=None):
52 data
['req'] = 'RpcApi' + method
53 data
= compat_urllib_parse_urlencode(data
).encode('utf-8')
54 return self
._download
_xml
(
55 'https://www.crunchyroll.com/xml/',
56 video_id
, note
, fatal
=False, data
=data
, headers
={
57 'Content-Type': 'application/x-www-form-urlencoded',
60 def _perform_login(self
, username
, password
):
61 if self
._get
_cookies
(self
._LOGIN
_URL
).get('etp_rt'):
64 upsell_response
= self
._download
_json
(
65 f
'{self._API_BASE}/get_upsell_data.0.json', None, 'Getting session id',
68 'device_id': 'whatvalueshouldbeforweb',
69 'device_type': 'com.crunchyroll.static',
70 'access_token': 'giKq5eY27ny3cqz',
71 'referer': self
._LOGIN
_URL
73 if upsell_response
['code'] != 'ok':
74 raise ExtractorError('Could not get session id')
75 session_id
= upsell_response
['data']['session_id']
77 login_response
= self
._download
_json
(
78 f
'{self._API_BASE}/login.1.json', None, 'Logging in',
79 data
=compat_urllib_parse_urlencode({
82 'session_id': session_id
84 if login_response
['code'] != 'ok':
85 raise ExtractorError('Login failed. Server message: %s' % login_response
['message'], expected
=True)
86 if not self
._get
_cookies
(self
._LOGIN
_URL
).get('etp_rt'):
87 raise ExtractorError('Login succeeded but did not set etp_rt cookie')
90 def _add_skip_wall(url
):
91 parsed_url
= compat_urlparse
.urlparse(url
)
92 qs
= compat_urlparse
.parse_qs(parsed_url
.query
)
93 # Always force skip_wall to bypass maturity wall, namely 18+ confirmation message:
94 # > This content may be inappropriate for some people.
95 # > Are you sure you want to continue?
96 # since it's not disabled by default in crunchyroll account's settings.
97 # See https://github.com/ytdl-org/youtube-dl/issues/7202.
98 qs
['skip_wall'] = ['1']
99 return compat_urlparse
.urlunparse(
100 parsed_url
._replace
(query
=compat_urllib_parse_urlencode(qs
, True)))
103 class CrunchyrollIE(CrunchyrollBaseIE
, VRVBaseIE
):
104 IE_NAME
= 'crunchyroll'
105 _VALID_URL
= r
'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|(?:[^/]*/){1,2}[^/?&]*?)(?P<id>[0-9]+))(?:[/?&]|$)'
107 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
111 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
112 'description': 'md5:2d17137920c64f2f49981a7797d275ef',
113 'thumbnail': r
're:^https?://.*\.jpg$',
114 'uploader': 'Yomiuri Telecasting Corporation (YTV)',
115 'upload_date': '20131013',
116 'url': 're:(?!.*&)',
120 'skip_download': True,
122 'skip': 'Video gone',
124 'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1',
128 'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
129 'description': 'md5:2fbc01f90b87e8e9137296f37b461c12',
130 'thumbnail': r
're:^https?://.*\.jpg$',
131 'uploader': 'Danny Choo Network',
132 'upload_date': '20120213',
136 'skip_download': True,
138 'skip': 'Video gone',
140 'url': 'http://www.crunchyroll.com/rezero-starting-life-in-another-world-/episode-5-the-morning-of-our-promise-is-still-distant-702409',
145 'description': compat_str
,
146 'thumbnail': r
're:^https?://.*\.jpg$',
147 'uploader': 'Re:Zero Partners',
148 'timestamp': 1462098900,
149 'upload_date': '20160501',
153 'skip_download': True,
156 'url': 'http://www.crunchyroll.com/konosuba-gods-blessing-on-this-wonderful-world/episode-1-give-me-deliverance-from-this-judicial-injustice-727589',
161 'description': compat_str
,
162 'thumbnail': r
're:^https?://.*\.jpg$',
163 'uploader': 'Kadokawa Pictures Inc.',
164 'timestamp': 1484130900,
165 'upload_date': '20170111',
166 'series': compat_str
,
167 'season': "KONOSUBA -God's blessing on this wonderful world! 2",
169 'episode': 'Give Me Deliverance From This Judicial Injustice!',
174 'skip_download': True,
177 'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
178 'only_matching': True,
180 # geo-restricted (US), 18+ maturity wall, non-premium available
181 'url': 'http://www.crunchyroll.com/cosplay-complex-ova/episode-1-the-birth-of-the-cosplay-club-565617',
182 'only_matching': True,
184 # A description with double quotes
185 'url': 'http://www.crunchyroll.com/11eyes/episode-1-piros-jszaka-red-night-535080',
190 'description': compat_str
,
191 'uploader': 'Marvelous AQL Inc.',
192 'timestamp': 1255512600,
193 'upload_date': '20091014',
196 # Just test metadata extraction
197 'skip_download': True,
200 # make sure we can extract an uploader name that's not a link
201 'url': 'http://www.crunchyroll.com/hakuoki-reimeiroku/episode-1-dawn-of-the-divine-warriors-606899',
205 'title': 'Hakuoki Reimeiroku Episode 1 – Dawn of the Divine Warriors',
206 'description': 'Ryunosuke was left to die, but Serizawa-san asked him a simple question "Do you want to live?"',
207 'uploader': 'Geneon Entertainment',
208 'upload_date': '20120717',
211 # just test metadata extraction
212 'skip_download': True,
214 'skip': 'Video gone',
216 # A video with a vastly different season name compared to the series name
217 'url': 'http://www.crunchyroll.com/nyarko-san-another-crawling-chaos/episode-1-test-590532',
222 'description': compat_str
,
223 'uploader': 'TV TOKYO',
224 'timestamp': 1330956000,
225 'upload_date': '20120305',
226 'series': 'Nyarko-san: Another Crawling Chaos',
227 'season': 'Haiyoru! Nyaruani (ONA)',
230 # Just test metadata extraction
231 'skip_download': True,
234 'url': 'http://www.crunchyroll.com/media-723735',
235 'only_matching': True,
237 'url': 'https://www.crunchyroll.com/en-gb/mob-psycho-100/episode-2-urban-legends-encountering-rumors-780921',
238 'only_matching': True,
242 '360': ('60', '106'),
243 '480': ('61', '106'),
244 '720': ('62', '106'),
245 '1080': ('80', '108'),
248 def _download_webpage(self
, url_or_request
, *args
, **kwargs
):
249 request
= (url_or_request
if isinstance(url_or_request
, compat_urllib_request
.Request
)
250 else sanitized_Request(url_or_request
))
251 # Accept-Language must be set explicitly to accept any language to avoid issues
252 # similar to https://github.com/ytdl-org/youtube-dl/issues/6797.
253 # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
254 # should be imposed or not (from what I can see it just takes the first language
255 # ignoring the priority and requires it to correspond the IP). By the way this causes
256 # Crunchyroll to not work in georestriction cases in some browsers that don't place
257 # the locale lang first in header. However allowing any language seems to workaround the issue.
258 request
.add_header('Accept-Language', '*')
259 return super(CrunchyrollBaseIE
, self
)._download
_webpage
(request
, *args
, **kwargs
)
261 def _decrypt_subtitles(self
, data
, iv
, id):
262 data
= bytes_to_intlist(compat_b64decode(data
))
263 iv
= bytes_to_intlist(compat_b64decode(iv
))
266 def obfuscate_key_aux(count
, modulo
, start
):
268 for _
in range(count
):
269 output
.append(output
[-1] + output
[-2])
270 # cut off start values
272 output
= list(map(lambda x
: x
% modulo
+ 33, output
))
275 def obfuscate_key(key
):
276 num1
= int(floor(pow(2, 25) * sqrt(6.9)))
277 num2
= (num1 ^ key
) << 5
279 num4
= num3 ^
(num3
>> 3) ^ num2
280 prefix
= intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2)))
281 shaHash
= bytes_to_intlist(sha1(prefix
+ str(num4
).encode('ascii')).digest())
282 # Extend 160 Bit hash to 256 Bit
283 return shaHash
+ [0] * 12
285 key
= obfuscate_key(id)
287 decrypted_data
= intlist_to_bytes(aes_cbc_decrypt(data
, key
, iv
))
288 return zlib
.decompress(decrypted_data
)
290 def _convert_subtitles_to_srt(self
, sub_root
):
293 for i
, event
in enumerate(sub_root
.findall('./events/event'), 1):
294 start
= event
.attrib
['start'].replace('.', ',')
295 end
= event
.attrib
['end'].replace('.', ',')
296 text
= event
.attrib
['text'].replace('\\N', '\n')
297 output
+= '%d\n%s --> %s\n%s\n\n' % (i
, start
, end
, text
)
300 def _convert_subtitles_to_ass(self
, sub_root
):
303 def ass_bool(strvalue
):
309 output
= '[Script Info]\n'
310 output
+= 'Title: %s\n' % sub_root
.attrib
['title']
311 output
+= 'ScriptType: v4.00+\n'
312 output
+= 'WrapStyle: %s\n' % sub_root
.attrib
['wrap_style']
313 output
+= 'PlayResX: %s\n' % sub_root
.attrib
['play_res_x']
314 output
+= 'PlayResY: %s\n' % sub_root
.attrib
['play_res_y']
317 Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
319 for style
in sub_root
.findall('./styles/style'):
320 output
+= 'Style: ' + style
.attrib
['name']
321 output
+= ',' + style
.attrib
['font_name']
322 output
+= ',' + style
.attrib
['font_size']
323 output
+= ',' + style
.attrib
['primary_colour']
324 output
+= ',' + style
.attrib
['secondary_colour']
325 output
+= ',' + style
.attrib
['outline_colour']
326 output
+= ',' + style
.attrib
['back_colour']
327 output
+= ',' + ass_bool(style
.attrib
['bold'])
328 output
+= ',' + ass_bool(style
.attrib
['italic'])
329 output
+= ',' + ass_bool(style
.attrib
['underline'])
330 output
+= ',' + ass_bool(style
.attrib
['strikeout'])
331 output
+= ',' + style
.attrib
['scale_x']
332 output
+= ',' + style
.attrib
['scale_y']
333 output
+= ',' + style
.attrib
['spacing']
334 output
+= ',' + style
.attrib
['angle']
335 output
+= ',' + style
.attrib
['border_style']
336 output
+= ',' + style
.attrib
['outline']
337 output
+= ',' + style
.attrib
['shadow']
338 output
+= ',' + style
.attrib
['alignment']
339 output
+= ',' + style
.attrib
['margin_l']
340 output
+= ',' + style
.attrib
['margin_r']
341 output
+= ',' + style
.attrib
['margin_v']
342 output
+= ',' + style
.attrib
['encoding']
347 Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
349 for event
in sub_root
.findall('./events/event'):
350 output
+= 'Dialogue: 0'
351 output
+= ',' + event
.attrib
['start']
352 output
+= ',' + event
.attrib
['end']
353 output
+= ',' + event
.attrib
['style']
354 output
+= ',' + event
.attrib
['name']
355 output
+= ',' + event
.attrib
['margin_l']
356 output
+= ',' + event
.attrib
['margin_r']
357 output
+= ',' + event
.attrib
['margin_v']
358 output
+= ',' + event
.attrib
['effect']
359 output
+= ',' + event
.attrib
['text']
364 def _extract_subtitles(self
, subtitle
):
365 sub_root
= compat_etree_fromstring(subtitle
)
368 'data': self
._convert
_subtitles
_to
_srt
(sub_root
),
371 'data': self
._convert
_subtitles
_to
_ass
(sub_root
),
374 def _get_subtitles(self
, video_id
, webpage
):
376 for sub_id
, sub_name
in re
.findall(r
'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage
):
377 sub_doc
= self
._call
_rpc
_api
(
378 'Subtitle_GetXml', video_id
,
379 'Downloading subtitles for ' + sub_name
, data
={
380 'subtitle_script_id': sub_id
,
382 if not isinstance(sub_doc
, compat_etree_Element
):
384 sid
= sub_doc
.get('id')
385 iv
= xpath_text(sub_doc
, 'iv', 'subtitle iv')
386 data
= xpath_text(sub_doc
, 'data', 'subtitle data')
387 if not sid
or not iv
or not data
:
389 subtitle
= self
._decrypt
_subtitles
(data
, iv
, sid
).decode('utf-8')
390 lang_code
= self
._search
_regex
(r
'lang_code=["\']([^
"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
393 subtitles[lang_code] = self._extract_subtitles(subtitle)
396 def _real_extract(self, url):
397 mobj = self._match_valid_url(url)
398 video_id = mobj.group('id')
400 if mobj.group('prefix') == 'm':
401 mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage')
402 webpage_url = self._search_regex(r'<link rel="canonical
" href="([^
"]+)" />', mobile_webpage, 'webpage_url
')
404 webpage_url = 'http
://www
.' + mobj.group('url
')
406 webpage = self._download_webpage(
407 self._add_skip_wall(webpage_url), video_id,
408 headers=self.geo_verification_headers())
409 note_m = self._html_search_regex(
410 r'<div
class="showmedia-trailer-notice">(.+?
)</div
>',
411 webpage, 'trailer
-notice
', default='')
413 raise ExtractorError(note_m, expected=True)
415 mobj = re.search(r'Page\
.messaging_box_controller\
.addItems\
(\
[(?P
<msg
>{.+?}
)\
]\
)', webpage)
417 msg = json.loads(mobj.group('msg
'))
418 if msg.get('type') == 'error
':
419 raise ExtractorError('crunchyroll returned error
: %s' % msg['message_body
'], expected=True)
421 if 'To view this
, please log
in to verify you are
18 or older
.' in webpage:
422 self.raise_login_required()
424 media = self._parse_json(self._search_regex(
425 r'vilos\
.config\
.media\s
*=\s
*({.+?}
);',
426 webpage, 'vilos media
', default='{}'), video_id)
427 media_metadata = media.get('metadata
') or {}
429 language = self._search_regex(
430 r'(?
:vilos\
.config\
.player\
.language|LOCALE
)\s
*=\s
*(["\'])(?P<lang>(?:(?!\1).)+)\1',
431 webpage, 'language', default=None, group='lang')
433 video_title = self._html_search_regex(
434 (r'(?s)<h1[^>]*>((?:(?!<h1).)*?<(?:span[^>]+itemprop=["\']title
["\']|meta[^>]+itemprop=["\']position
["\'])[^>]*>(?:(?!<h1).)+?)</h1>',
435 r'<title>(.+?),\s+-\s+.+? Crunchyroll'),
436 webpage, 'video_title', default=None)
438 video_title = re.sub(r'^Watch\s+', '', self._og_search_description(webpage))
439 video_title = re.sub(r' {2,}', ' ', video_title)
440 video_description = (self._parse_json(self._html_search_regex(
441 r'<script[^>]*>\s*.+?\[media_id=%s\].+?({.+?"description"\s*:.+?})\);' % video_id,
442 webpage, 'description', default='{}'), video_id) or media_metadata).get('description')
445 thumbnail_url = (self._parse_json(self._html_search_regex(
446 r'<script type="application\
/ld\
+json
">\n\s*(.+?)<\/script>',
447 webpage, 'thumbnail_url', default='{}'), video_id)).get('image')
450 'url': thumbnail_url,
455 if video_description:
456 video_description = lowercase_escape(video_description.replace(r'\r\n', '\n'))
457 video_uploader = self._html_search_regex(
458 # try looking for both an uploader that's a link and one that's not
459 [r'<a[^>]+href="/publisher
/[^
"]+"[^
>]*>([^
<]+)</a
>', r'<div
>\s
*Publisher
:\s
*<span
>\s
*(.+?
)\s
*</span
>\s
*</div
>'],
460 webpage, 'video_uploader
', default=False)
462 requested_languages = self._configuration_arg('language
')
463 requested_hardsubs = [('' if val == 'none
' else val) for val in self._configuration_arg('hardsub
')]
464 language_preference = qualities((requested_languages or [language or ''])[::-1])
465 hardsub_preference = qualities((requested_hardsubs or ['', language or ''])[::-1])
468 for stream in media.get('streams
', []):
469 audio_lang = stream.get('audio_lang
') or ''
470 hardsub_lang = stream.get('hardsub_lang
') or ''
471 if (requested_languages and audio_lang.lower() not in requested_languages
472 or requested_hardsubs and hardsub_lang.lower() not in requested_hardsubs):
474 vrv_formats = self._extract_vrv_formats(
475 stream.get('url
'), video_id, stream.get('format
'),
476 audio_lang, hardsub_lang)
477 for f in vrv_formats:
478 f['language_preference
'] = language_preference(audio_lang)
479 f['quality
'] = hardsub_preference(hardsub_lang)
480 formats.extend(vrv_formats)
483 for a, fmt in re.findall(r'(<a
[^
>]+token
=["\']showmedia\.([0-9]{3,4})p["\'][^
>]+>)', webpage):
484 attrs = extract_attributes(a)
485 href = attrs.get('href
')
486 if href and '/freetrial
' in href:
488 available_fmts.append(fmt)
489 if not available_fmts:
490 for p in (r'token
=["\']showmedia\.([0-9]{3,4})p"', r'showmedia\
.([0-9]{3,4}
)p
'):
491 available_fmts = re.findall(p, webpage)
494 if not available_fmts:
495 available_fmts = self._FORMAT_IDS.keys()
496 video_encode_ids = []
498 for fmt in available_fmts:
499 stream_quality, stream_format = self._FORMAT_IDS[fmt]
500 video_format = fmt + 'p
'
502 streamdata = self._call_rpc_api(
503 'VideoPlayer_GetStandardConfig
', video_id,
504 'Downloading media info
for %s' % video_format, data={
505 'media_id
': video_id,
506 'video_format
': stream_format,
507 'video_quality
': stream_quality,
510 if isinstance(streamdata, compat_etree_Element):
511 stream_info = streamdata.find('./{default}preload
/stream_info
')
512 if stream_info is not None:
513 stream_infos.append(stream_info)
514 stream_info = self._call_rpc_api(
515 'VideoEncode_GetStreamInfo
', video_id,
516 'Downloading stream info
for %s' % video_format, data={
517 'media_id
': video_id,
518 'video_format
': stream_format,
519 'video_encode_quality
': stream_quality,
521 if isinstance(stream_info, compat_etree_Element):
522 stream_infos.append(stream_info)
523 for stream_info in stream_infos:
524 video_encode_id = xpath_text(stream_info, './video_encode_id
')
525 if video_encode_id in video_encode_ids:
527 video_encode_ids.append(video_encode_id)
529 video_file = xpath_text(stream_info, './file')
532 if video_file.startswith('http
'):
533 formats.extend(self._extract_m3u8_formats(
534 video_file, video_id, 'mp4
', entry_protocol='m3u8_native
',
535 m3u8_id='hls
', fatal=False))
538 video_url = xpath_text(stream_info, './host
')
541 metadata = stream_info.find('./metadata
')
543 'format
': video_format,
544 'height
': int_or_none(xpath_text(metadata, './height
')),
545 'width
': int_or_none(xpath_text(metadata, './width
')),
548 if '.fplive
.net
/' in video_url:
549 video_url = re.sub(r'^rtmpe?
://', 'http
://', video_url.strip())
550 parsed_video_url = compat_urlparse.urlparse(video_url)
551 direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
552 netloc='v
.lvlt
.crcdn
.net
',
553 path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
554 if self._is_valid_url(direct_video_url, video_id, video_format):
556 'format_id
': 'http
-' + video_format,
557 'url
': direct_video_url,
559 formats.append(format_info)
563 'format_id
': 'rtmp
-' + video_format,
565 'play_path
': video_file,
568 formats.append(format_info)
569 self._sort_formats(formats)
571 metadata = self._call_rpc_api(
572 'VideoPlayer_GetMediaMetadata
', video_id,
573 note='Downloading media info
', data={
574 'media_id
': video_id,
578 for subtitle in media.get('subtitles
', []):
579 subtitle_url = subtitle.get('url
')
582 subtitles.setdefault(subtitle.get('language
', 'enUS
'), []).append({
584 'ext
': subtitle.get('format
', 'ass
'),
587 subtitles = self.extract_subtitles(video_id, webpage)
589 # webpage provide more accurate data than series_title from XML
590 series = self._html_search_regex(
591 r'(?s
)<h\d
[^
>]+\bid
=["\']showmedia_about_episode_num[^>]+>(.+?)</h\d',
592 webpage, 'series', fatal=False)
594 season = episode = episode_number = duration = None
596 if isinstance(metadata, compat_etree_Element):
597 season = xpath_text(metadata, 'series_title')
598 episode = xpath_text(metadata, 'episode_title')
599 episode_number = int_or_none(xpath_text(metadata, 'episode_number'))
600 duration = float_or_none(media_metadata.get('duration'), 1000)
603 episode = media_metadata.get('title')
604 if not episode_number:
605 episode_number = int_or_none(media_metadata.get('episode_number'))
606 thumbnail_url = try_get(media, lambda x: x['thumbnail']['url'])
609 'url': thumbnail_url,
614 season_number = int_or_none(self._search_regex(
615 r'(?s)<h\d[^>]+id=["\']showmedia_about_episode_num
[^
>]+>.+?
</h\d
>\s
*<h4
>\s
*Season (\d
+)',
616 webpage, 'season number
', default=None))
618 info = self._search_json_ld(webpage, video_id, default={})
622 'title
': video_title,
623 'description
': video_description,
624 'duration
': duration,
625 'thumbnails
': thumbnails,
626 'uploader
': video_uploader,
629 'season_number
': season_number,
631 'episode_number
': episode_number,
632 'subtitles
': subtitles,
637 class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
638 IE_NAME = 'crunchyroll
:playlist
'
639 _VALID_URL = r'https?
://(?
:(?P
<prefix
>www|m
)\
.)?
(?P
<url
>crunchyroll\
.com
/(?
:\w{1,2}
/)?
(?
!(?
:news|anime
-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login|media
-\d
+))(?P
<id>[\w\
-]+))/?
(?
:\?|$
)'
642 'url
': 'https
://www
.crunchyroll
.com
/a
-bridge
-to
-the
-starry
-skies
-hoshizora
-e
-kakaru
-hashi
',
644 'id': 'a
-bridge
-to
-the
-starry
-skies
-hoshizora
-e
-kakaru
-hashi
',
645 'title
': 'A Bridge to the Starry Skies
- Hoshizora e Kakaru Hashi
'
647 'playlist_count
': 13,
649 # geo-restricted (US), 18+ maturity wall, non-premium available
650 'url
': 'http
://www
.crunchyroll
.com
/cosplay
-complex-ova
',
652 'id': 'cosplay
-complex-ova
',
653 'title
': 'Cosplay Complex OVA
'
656 'skip
': 'Georestricted
',
658 # geo-restricted (US), 18+ maturity wall, non-premium will be available since 2015.11.14
659 'url
': 'http
://www
.crunchyroll
.com
/ladies
-versus
-butlers?skip_wall
=1',
660 'only_matching
': True,
662 'url
': 'http
://www
.crunchyroll
.com
/fr
/ladies
-versus
-butlers
',
663 'only_matching
': True,
666 def _real_extract(self, url):
667 show_id = self._match_id(url)
669 webpage = self._download_webpage(
670 # https:// gives a 403, but http:// does not
671 self._add_skip_wall(url).replace('https
://', 'http
://'), show_id,
672 headers=self.geo_verification_headers())
673 title = self._html_search_meta('name
', webpage, default=None)
675 episode_re = r'<li
id="showview_videos_media_(\d+)"[^
>]+>.*?
<a href
="([^"]+)"'
676 season_re = r'<a [^>]+season-dropdown[^>]+>([^<]+)'
677 paths = re.findall(f'(?s){episode_re}|{season_re}', webpage)
679 entries, current_season = [], None
680 for ep_id, ep, season in paths:
682 current_season = season
684 entries.append(self.url_result(
685 f'http://www.crunchyroll.com{ep}', CrunchyrollIE.ie_key(), ep_id, season=current_season))
691 'entries': reversed(entries),
695 class CrunchyrollBetaIE(CrunchyrollBaseIE):
696 IE_NAME = 'crunchyroll:beta'
697 _VALID_URL = r'https?://beta\.crunchyroll\.com/(?P<lang>(?:\w{1,2}/)?)watch/(?P<internal_id>\w+)/(?P<id>[\w\-]+)/?(?:\?|$)'
699 'url': 'https://beta.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
703 'timestamp': 1459610100,
704 'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
705 'uploader': 'Toei Animation',
706 'title': 'World Trigger Episode 73 – To the Future',
707 'upload_date': '20160402',
709 'params': {'skip_download': 'm3u8'},
710 'expected_warnings': ['Unable to download XML']
713 def _real_extract(self, url):
714 lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'internal_id', 'id')
715 webpage = self._download_webpage(url, display_id)
716 initial_state = self._parse_json(
717 self._search_regex(r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'initial state'),
719 episode_data = initial_state['content']['byId'][internal_id]
720 if not self._get_cookies(url).get('etp_rt'):
721 video_id = episode_data['external_id'].split('.')[1]
722 series_id = episode_data['episode_metadata']['series_slug_title']
723 return self.url_result(f'https://www.crunchyroll.com/{lang}{series_id}/{display_id}-{video_id}',
724 CrunchyrollIE.ie_key(), video_id)
726 app_config = self._parse_json(
727 self._search_regex(r'__APP_CONFIG__\s*=\s*({.+?})\s*;', webpage, 'app config'),
729 client_id = app_config['cxApiParams']['accountAuthClientId']
730 api_domain = app_config['cxApiParams']['apiDomain']
731 basic_token = str(base64.b64encode(('%s:' % client_id).encode('ascii')), 'ascii')
732 auth_response = self._download_json(
733 f'{api_domain}/auth/v1/token', display_id,
734 note='Authenticating with cookie',
736 'Authorization': 'Basic ' + basic_token
737 }, data='grant_type=etp_rt_cookie'.encode('ascii'))
738 policy_response = self._download_json(
739 f'{api_domain}/index/v2', display_id,
740 note='Retrieving signed policy',
742 'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']
744 bucket = policy_response['cms']['bucket']
746 'Policy': policy_response['cms']['policy'],
747 'Signature': policy_response['cms']['signature'],
748 'Key-Pair-Id': policy_response['cms']['key_pair_id']
750 locale = traverse_obj(initial_state, ('localization', 'locale'))
752 params['locale'] = locale
753 episode_response = self._download_json(
754 f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
755 note='Retrieving episode metadata',
757 if episode_response.get('is_premium_only') and not episode_response.get('playback'):
758 raise ExtractorError('This video is for premium members only.', expected=True)
759 stream_response = self._download_json(
760 episode_response['playback'], display_id,
761 note='Retrieving stream info')
764 for thumbnails_data in traverse_obj(episode_response, ('images', 'thumbnail')):
765 for thumbnail_data in thumbnails_data:
767 'url': thumbnail_data.get('source'),
768 'width': thumbnail_data.get('width'),
769 'height': thumbnail_data.get('height'),
772 for lang, subtitle_data in stream_response.get('subtitles').items():
774 'url': subtitle_data.get('url'),
775 'ext': subtitle_data.get('format')
778 requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
779 hardsub_preference = qualities(requested_hardsubs[::-1])
780 requested_formats = self._configuration_arg('format') or ['adaptive_hls']
783 for stream_type, streams in stream_response.get('streams', {}).items():
784 if stream_type not in requested_formats:
786 for stream in streams.values():
787 hardsub_lang = stream.get('hardsub_locale') or ''
788 if hardsub_lang.lower() not in requested_hardsubs:
790 format_id = join_nonempty(
792 format_field(stream, 'hardsub_locale', 'hardsub-%s'))
793 if not stream.get('url'):
795 if stream_type.split('_')[-1] == 'hls':
796 adaptive_formats = self._extract_m3u8_formats(
797 stream['url'], display_id, 'mp4', m3u8_id=format_id,
798 note='Downloading %s information' % format_id,
800 elif stream_type.split('_')[-1] == 'dash':
801 adaptive_formats = self._extract_mpd_formats(
802 stream['url'], display_id, mpd_id=format_id,
803 note='Downloading %s information' % format_id,
805 for f in adaptive_formats:
806 if f.get('acodec') != 'none':
807 f['language'] = stream_response.get('audio_locale')
808 f['quality'] = hardsub_preference(hardsub_lang.lower())
809 formats.extend(adaptive_formats)
810 self._sort_formats(formats)
814 'title': '%s Episode %s – %s' % (episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
815 'description': episode_response.get('description').replace(r'\r\n', '\n'),
816 'duration': float_or_none(episode_response.get('duration_ms'), 1000),
817 'thumbnails': thumbnails,
818 'series': episode_response.get('series_title'),
819 'series_id': episode_response.get('series_id'),
820 'season': episode_response.get('season_title'),
821 'season_id': episode_response.get('season_id'),
822 'season_number': episode_response.get('season_number'),
823 'episode': episode_response.get('title'),
824 'episode_number': episode_response.get('sequence_number'),
825 'subtitles': subtitles,
830 class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
831 IE_NAME = 'crunchyroll:playlist:beta'
832 _VALID_URL = r'https?://beta\.crunchyroll\.com/(?P<lang>(?:\w{1,2}/)?)series/\w+/(?P<id>[\w\-]+)/?(?:\?|$)'
834 'url': 'https://beta.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA',
836 'id': 'girl-friend-beta',
837 'title': 'Girl Friend BETA',
839 'playlist_mincount': 10,
841 'url': 'https://beta.crunchyroll.com/it/series/GY19NQ2QR/Girl-Friend-BETA',
842 'only_matching': True,
845 def _real_extract(self, url):
846 lang, series_id = self._match_valid_url(url).group('lang', 'id')
847 return self.url_result(f'https://www.crunchyroll.com/{lang}{series_id.lower()}',
848 CrunchyrollShowPlaylistIE.ie_key(), series_id)