2 from __future__
import unicode_literals
8 import xml
.etree
.ElementTree
10 from hashlib
import sha1
11 from math
import pow, sqrt
, floor
12 from .common
import InfoExtractor
13 from ..compat
import (
15 compat_urllib_parse_unquote
,
16 compat_urllib_request
,
34 class CrunchyrollIE(InfoExtractor
):
35 _VALID_URL
= r
'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
36 _NETRC_MACHINE
= 'crunchyroll'
38 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
42 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
43 'description': 'md5:2d17137920c64f2f49981a7797d275ef',
44 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
45 'uploader': 'Yomiuri Telecasting Corporation (YTV)',
46 'upload_date': '20131013',
47 'url': 're:(?!.*&)',
51 'skip_download': True,
54 'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1',
58 'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
59 'description': 'md5:fe2743efedb49d279552926d0bd0cd9e',
60 'thumbnail': 're:^https?://.*\.jpg$',
61 'uploader': 'Danny Choo Network',
62 'upload_date': '20120213',
66 'skip_download': True,
70 'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
71 'only_matching': True,
78 '1080': ('80', '108'),
82 (username
, password
) = self
._get
_login
_info
()
86 login_url
= 'https://www.crunchyroll.com/?a=formhandler'
87 data
= urlencode_postdata({
88 'formname': 'RpcApiUser_Login',
92 login_request
= compat_urllib_request
.Request(login_url
, data
)
93 login_request
.add_header('Content-Type', 'application/x-www-form-urlencoded')
94 self
._download
_webpage
(login_request
, None, False, 'Wrong login info')
96 def _real_initialize(self
):
99 def _decrypt_subtitles(self
, data
, iv
, id):
100 data
= bytes_to_intlist(base64
.b64decode(data
.encode('utf-8')))
101 iv
= bytes_to_intlist(base64
.b64decode(iv
.encode('utf-8')))
104 def obfuscate_key_aux(count
, modulo
, start
):
106 for _
in range(count
):
107 output
.append(output
[-1] + output
[-2])
108 # cut off start values
110 output
= list(map(lambda x
: x
% modulo
+ 33, output
))
113 def obfuscate_key(key
):
114 num1
= int(floor(pow(2, 25) * sqrt(6.9)))
115 num2
= (num1 ^ key
) << 5
117 num4
= num3 ^
(num3
>> 3) ^ num2
118 prefix
= intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2)))
119 shaHash
= bytes_to_intlist(sha1(prefix
+ str(num4
).encode('ascii')).digest())
120 # Extend 160 Bit hash to 256 Bit
121 return shaHash
+ [0] * 12
123 key
= obfuscate_key(id)
125 decrypted_data
= intlist_to_bytes(aes_cbc_decrypt(data
, key
, iv
))
126 return zlib
.decompress(decrypted_data
)
128 def _convert_subtitles_to_srt(self
, sub_root
):
131 for i
, event
in enumerate(sub_root
.findall('./events/event'), 1):
132 start
= event
.attrib
['start'].replace('.', ',')
133 end
= event
.attrib
['end'].replace('.', ',')
134 text
= event
.attrib
['text'].replace('\\N', '\n')
135 output
+= '%d\n%s --> %s\n%s\n\n' % (i
, start
, end
, text
)
138 def _convert_subtitles_to_ass(self
, sub_root
):
141 def ass_bool(strvalue
):
147 output
= '[Script Info]\n'
148 output
+= 'Title: %s\n' % sub_root
.attrib
["title"]
149 output
+= 'ScriptType: v4.00+\n'
150 output
+= 'WrapStyle: %s\n' % sub_root
.attrib
["wrap_style"]
151 output
+= 'PlayResX: %s\n' % sub_root
.attrib
["play_res_x"]
152 output
+= 'PlayResY: %s\n' % sub_root
.attrib
["play_res_y"]
153 output
+= """ScaledBorderAndShadow: yes
156 Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
158 for style
in sub_root
.findall('./styles/style'):
159 output
+= 'Style: ' + style
.attrib
["name"]
160 output
+= ',' + style
.attrib
["font_name"]
161 output
+= ',' + style
.attrib
["font_size"]
162 output
+= ',' + style
.attrib
["primary_colour"]
163 output
+= ',' + style
.attrib
["secondary_colour"]
164 output
+= ',' + style
.attrib
["outline_colour"]
165 output
+= ',' + style
.attrib
["back_colour"]
166 output
+= ',' + ass_bool(style
.attrib
["bold"])
167 output
+= ',' + ass_bool(style
.attrib
["italic"])
168 output
+= ',' + ass_bool(style
.attrib
["underline"])
169 output
+= ',' + ass_bool(style
.attrib
["strikeout"])
170 output
+= ',' + style
.attrib
["scale_x"]
171 output
+= ',' + style
.attrib
["scale_y"]
172 output
+= ',' + style
.attrib
["spacing"]
173 output
+= ',' + style
.attrib
["angle"]
174 output
+= ',' + style
.attrib
["border_style"]
175 output
+= ',' + style
.attrib
["outline"]
176 output
+= ',' + style
.attrib
["shadow"]
177 output
+= ',' + style
.attrib
["alignment"]
178 output
+= ',' + style
.attrib
["margin_l"]
179 output
+= ',' + style
.attrib
["margin_r"]
180 output
+= ',' + style
.attrib
["margin_v"]
181 output
+= ',' + style
.attrib
["encoding"]
186 Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
188 for event
in sub_root
.findall('./events/event'):
189 output
+= 'Dialogue: 0'
190 output
+= ',' + event
.attrib
["start"]
191 output
+= ',' + event
.attrib
["end"]
192 output
+= ',' + event
.attrib
["style"]
193 output
+= ',' + event
.attrib
["name"]
194 output
+= ',' + event
.attrib
["margin_l"]
195 output
+= ',' + event
.attrib
["margin_r"]
196 output
+= ',' + event
.attrib
["margin_v"]
197 output
+= ',' + event
.attrib
["effect"]
198 output
+= ',' + event
.attrib
["text"]
203 def _extract_subtitles(self
, subtitle
):
204 sub_root
= xml
.etree
.ElementTree
.fromstring(subtitle
)
207 'data': self
._convert
_subtitles
_to
_srt
(sub_root
),
210 'data': self
._convert
_subtitles
_to
_ass
(sub_root
),
213 def _get_subtitles(self
, video_id
, webpage
):
215 for sub_id
, sub_name
in re
.findall(r
'\?ssid=([0-9]+)" title="([^"]+)', webpage
):
216 sub_page
= self
._download
_webpage
(
217 'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id
,
218 video_id
, note
='Downloading subtitles for ' + sub_name
)
219 id = self
._search
_regex
(r
'id=\'([0-9]+)', sub_page, 'subtitle_id
', fatal=False)
220 iv = self._search_regex(r'<iv
>([^
<]+)', sub_page, 'subtitle_iv
', fatal=False)
221 data = self._search_regex(r'<data
>([^
<]+)', sub_page, 'subtitle_data
', fatal=False)
222 if not id or not iv or not data:
224 subtitle = self._decrypt_subtitles(data, iv, id).decode('utf
-8')
225 lang_code = self._search_regex(r'lang_code
=["\']([^"\']+)', subtitle, 'subtitle_lang_code
', fatal=False)
228 subtitles[lang_code] = self._extract_subtitles(subtitle)
231 def _real_extract(self, url):
232 mobj = re.match(self._VALID_URL, url)
233 video_id = mobj.group('video_id
')
235 if mobj.group('prefix
') == 'm
':
236 mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage
')
237 webpage_url = self._search_regex(r'<link rel
="canonical" href
="([^"]+)" />', mobile_webpage, 'webpage_url')
239 webpage_url = 'http://www.' + mobj.group('url')
241 webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
242 note_m = self._html_search_regex(
243 r'<div class="showmedia
-trailer
-notice
">(.+?)</div>',
244 webpage, 'trailer-notice', default='')
246 raise ExtractorError(note_m)
248 mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage)
250 msg = json.loads(mobj.group('msg'))
251 if msg.get('type') == 'error':
252 raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
254 if 'To view this, please log in to verify you are 18 or older.' in webpage:
255 self.raise_login_required()
257 video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
258 video_title = re.sub(r' {2,}', ' ', video_title)
259 video_description = self._html_search_regex(r'"description
":"([^
"]+)', webpage, 'video_description', default='')
260 if not video_description:
261 video_description = None
262 video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
263 if video_upload_date:
264 video_upload_date = unified_strdate(video_upload_date)
265 video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
267 playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url
":"([^
"]+)', webpage, 'playerdata_url'))
268 playerdata_req = compat_urllib_request.Request(playerdata_url)
269 playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
270 playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
271 playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
273 stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id')
274 video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
277 for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
278 stream_quality, stream_format = self._FORMAT_IDS[fmt]
279 video_format = fmt + 'p'
280 streamdata_req = compat_urllib_request.Request(
281 'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
282 % (stream_id, stream_format, stream_quality),
283 compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
284 streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
285 streamdata = self._download_xml(
286 streamdata_req, video_id,
287 note='Downloading media info for %s' % video_format)
288 stream_info = streamdata.find('./{default}preload/stream_info')
289 video_url = stream_info.find('./host').text
290 video_play_path = stream_info.find('./file').text
291 metadata = stream_info.find('./metadata')
293 'format': video_format,
294 'format_id': video_format,
295 'height': int_or_none(xpath_text(metadata, './height')),
296 'width': int_or_none(xpath_text(metadata, './width')),
299 if '.fplive.net/' in video_url:
300 video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
301 parsed_video_url = compat_urlparse.urlparse(video_url)
302 direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
303 netloc='v.lvlt.crcdn.net',
304 path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_play_path.split(':')[-1])))
305 if self._is_valid_url(direct_video_url, video_id, video_format):
307 'url': direct_video_url,
309 formats.append(format_info)
314 'play_path': video_play_path,
317 formats.append(format_info)
319 subtitles = self.extract_subtitles(video_id, webpage)
323 'title': video_title,
324 'description': video_description,
325 'thumbnail': video_thumbnail,
326 'uploader': video_uploader,
327 'upload_date': video_upload_date,
328 'subtitles': subtitles,
333 class CrunchyrollShowPlaylistIE(InfoExtractor):
334 IE_NAME = "crunchyroll
:playlist
"
335 _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
338 'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
340 'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
341 'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
343 'playlist_count': 13,
346 def _real_extract(self, url):
347 show_id = self._match_id(url)
349 webpage = self._download_webpage(url, show_id)
350 title = self._html_search_regex(
351 r'(?s)<h1[^>]*>\s*<span itemprop="name
">(.*?)</span>',
353 episode_paths = re.findall(
354 r'(?s)<li id="showview_videos_media_
[0-9]+"[^>]+>.*?<a href="([^
"]+)"',
357 self.url_result('http
://www
.crunchyroll
.com
' + ep, 'Crunchyroll
')
358 for ep in episode_paths