2 from __future__
import unicode_literals
8 import xml
.etree
.ElementTree
10 from hashlib
import sha1
11 from math
import pow, sqrt
, floor
12 from .common
import InfoExtractor
13 from ..compat
import (
15 compat_urllib_parse_unquote
,
16 compat_urllib_request
,
32 class CrunchyrollIE(InfoExtractor
):
33 _VALID_URL
= r
'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
34 _NETRC_MACHINE
= 'crunchyroll'
36 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
40 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
41 'description': 'md5:2d17137920c64f2f49981a7797d275ef',
42 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
43 'uploader': 'Yomiuri Telecasting Corporation (YTV)',
44 'upload_date': '20131013',
45 'url': 're:(?!.*&)',
49 'skip_download': True,
52 'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1',
56 'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
57 'description': 'md5:fe2743efedb49d279552926d0bd0cd9e',
58 'thumbnail': 're:^https?://.*\.jpg$',
59 'uploader': 'Danny Choo Network',
60 'upload_date': '20120213',
64 'skip_download': True,
68 'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
69 'only_matching': True,
76 '1080': ('80', '108'),
80 (username
, password
) = self
._get
_login
_info
()
84 login_url
= 'https://www.crunchyroll.com/?a=formhandler'
85 data
= urlencode_postdata({
86 'formname': 'RpcApiUser_Login',
90 login_request
= compat_urllib_request
.Request(login_url
, data
)
91 login_request
.add_header('Content-Type', 'application/x-www-form-urlencoded')
92 self
._download
_webpage
(login_request
, None, False, 'Wrong login info')
94 def _real_initialize(self
):
97 def _decrypt_subtitles(self
, data
, iv
, id):
98 data
= bytes_to_intlist(base64
.b64decode(data
.encode('utf-8')))
99 iv
= bytes_to_intlist(base64
.b64decode(iv
.encode('utf-8')))
102 def obfuscate_key_aux(count
, modulo
, start
):
104 for _
in range(count
):
105 output
.append(output
[-1] + output
[-2])
106 # cut off start values
108 output
= list(map(lambda x
: x
% modulo
+ 33, output
))
111 def obfuscate_key(key
):
112 num1
= int(floor(pow(2, 25) * sqrt(6.9)))
113 num2
= (num1 ^ key
) << 5
115 num4
= num3 ^
(num3
>> 3) ^ num2
116 prefix
= intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2)))
117 shaHash
= bytes_to_intlist(sha1(prefix
+ str(num4
).encode('ascii')).digest())
118 # Extend 160 Bit hash to 256 Bit
119 return shaHash
+ [0] * 12
121 key
= obfuscate_key(id)
123 decrypted_data
= intlist_to_bytes(aes_cbc_decrypt(data
, key
, iv
))
124 return zlib
.decompress(decrypted_data
)
126 def _convert_subtitles_to_srt(self
, sub_root
):
129 for i
, event
in enumerate(sub_root
.findall('./events/event'), 1):
130 start
= event
.attrib
['start'].replace('.', ',')
131 end
= event
.attrib
['end'].replace('.', ',')
132 text
= event
.attrib
['text'].replace('\\N', '\n')
133 output
+= '%d\n%s --> %s\n%s\n\n' % (i
, start
, end
, text
)
136 def _convert_subtitles_to_ass(self
, sub_root
):
139 def ass_bool(strvalue
):
145 output
= '[Script Info]\n'
146 output
+= 'Title: %s\n' % sub_root
.attrib
["title"]
147 output
+= 'ScriptType: v4.00+\n'
148 output
+= 'WrapStyle: %s\n' % sub_root
.attrib
["wrap_style"]
149 output
+= 'PlayResX: %s\n' % sub_root
.attrib
["play_res_x"]
150 output
+= 'PlayResY: %s\n' % sub_root
.attrib
["play_res_y"]
151 output
+= """ScaledBorderAndShadow: yes
154 Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
156 for style
in sub_root
.findall('./styles/style'):
157 output
+= 'Style: ' + style
.attrib
["name"]
158 output
+= ',' + style
.attrib
["font_name"]
159 output
+= ',' + style
.attrib
["font_size"]
160 output
+= ',' + style
.attrib
["primary_colour"]
161 output
+= ',' + style
.attrib
["secondary_colour"]
162 output
+= ',' + style
.attrib
["outline_colour"]
163 output
+= ',' + style
.attrib
["back_colour"]
164 output
+= ',' + ass_bool(style
.attrib
["bold"])
165 output
+= ',' + ass_bool(style
.attrib
["italic"])
166 output
+= ',' + ass_bool(style
.attrib
["underline"])
167 output
+= ',' + ass_bool(style
.attrib
["strikeout"])
168 output
+= ',' + style
.attrib
["scale_x"]
169 output
+= ',' + style
.attrib
["scale_y"]
170 output
+= ',' + style
.attrib
["spacing"]
171 output
+= ',' + style
.attrib
["angle"]
172 output
+= ',' + style
.attrib
["border_style"]
173 output
+= ',' + style
.attrib
["outline"]
174 output
+= ',' + style
.attrib
["shadow"]
175 output
+= ',' + style
.attrib
["alignment"]
176 output
+= ',' + style
.attrib
["margin_l"]
177 output
+= ',' + style
.attrib
["margin_r"]
178 output
+= ',' + style
.attrib
["margin_v"]
179 output
+= ',' + style
.attrib
["encoding"]
184 Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
186 for event
in sub_root
.findall('./events/event'):
187 output
+= 'Dialogue: 0'
188 output
+= ',' + event
.attrib
["start"]
189 output
+= ',' + event
.attrib
["end"]
190 output
+= ',' + event
.attrib
["style"]
191 output
+= ',' + event
.attrib
["name"]
192 output
+= ',' + event
.attrib
["margin_l"]
193 output
+= ',' + event
.attrib
["margin_r"]
194 output
+= ',' + event
.attrib
["margin_v"]
195 output
+= ',' + event
.attrib
["effect"]
196 output
+= ',' + event
.attrib
["text"]
201 def _extract_subtitles(self
, subtitle
):
202 sub_root
= xml
.etree
.ElementTree
.fromstring(subtitle
)
205 'data': self
._convert
_subtitles
_to
_srt
(sub_root
),
208 'data': self
._convert
_subtitles
_to
_ass
(sub_root
),
211 def _get_subtitles(self
, video_id
, webpage
):
213 for sub_id
, sub_name
in re
.findall(r
'\?ssid=([0-9]+)" title="([^"]+)', webpage
):
214 sub_page
= self
._download
_webpage
(
215 'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id
,
216 video_id
, note
='Downloading subtitles for ' + sub_name
)
217 id = self
._search
_regex
(r
'id=\'([0-9]+)', sub_page, 'subtitle_id
', fatal=False)
218 iv = self._search_regex(r'<iv
>([^
<]+)', sub_page, 'subtitle_iv
', fatal=False)
219 data = self._search_regex(r'<data
>([^
<]+)', sub_page, 'subtitle_data
', fatal=False)
220 if not id or not iv or not data:
222 subtitle = self._decrypt_subtitles(data, iv, id).decode('utf
-8')
223 lang_code = self._search_regex(r'lang_code
=["\']([^"\']+)', subtitle, 'subtitle_lang_code
', fatal=False)
226 subtitles[lang_code] = self._extract_subtitles(subtitle)
229 def _real_extract(self, url):
230 mobj = re.match(self._VALID_URL, url)
231 video_id = mobj.group('video_id
')
233 if mobj.group('prefix
') == 'm
':
234 mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage
')
235 webpage_url = self._search_regex(r'<link rel
="canonical" href
="([^"]+)" />', mobile_webpage, 'webpage_url')
237 webpage_url = 'http://www.' + mobj.group('url')
239 webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
240 note_m = self._html_search_regex(
241 r'<div class="showmedia
-trailer
-notice
">(.+?)</div>',
242 webpage, 'trailer-notice', default='')
244 raise ExtractorError(note_m)
246 mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage)
248 msg = json.loads(mobj.group('msg'))
249 if msg.get('type') == 'error':
250 raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
252 if 'To view this, please log in to verify you are 18 or older.' in webpage:
253 self.raise_login_required(video_id)
255 video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
256 video_title = re.sub(r' {2,}', ' ', video_title)
257 video_description = self._html_search_regex(r'"description
":"([^
"]+)', webpage, 'video_description', default='')
258 if not video_description:
259 video_description = None
260 video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
261 if video_upload_date:
262 video_upload_date = unified_strdate(video_upload_date)
263 video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
265 playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url
":"([^
"]+)', webpage, 'playerdata_url'))
266 playerdata_req = compat_urllib_request.Request(playerdata_url)
267 playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
268 playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
269 playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
271 stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id')
272 video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
275 for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
276 stream_quality, stream_format = self._FORMAT_IDS[fmt]
277 video_format = fmt + 'p'
278 streamdata_req = compat_urllib_request.Request(
279 'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
280 % (stream_id, stream_format, stream_quality),
281 compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
282 streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
283 streamdata = self._download_xml(
284 streamdata_req, video_id,
285 note='Downloading media info for %s' % video_format)
286 stream_info = streamdata.find('./{default}preload/stream_info')
287 video_url = stream_info.find('./host').text
288 video_play_path = stream_info.find('./file').text
290 if '.fplive.net/' in video_url:
291 video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
292 parsed_video_url = compat_urlparse.urlparse(video_url)
293 direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
294 netloc='v.lvlt.crcdn.net',
295 path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_play_path.split(':')[-1])))
296 if self._is_valid_url(direct_video_url, video_id, video_format):
298 'url': direct_video_url,
299 'format_id': video_format,
305 'play_path': video_play_path,
307 'format': video_format,
308 'format_id': video_format,
311 subtitles = self.extract_subtitles(video_id, webpage)
315 'title': video_title,
316 'description': video_description,
317 'thumbnail': video_thumbnail,
318 'uploader': video_uploader,
319 'upload_date': video_upload_date,
320 'subtitles': subtitles,
325 class CrunchyrollShowPlaylistIE(InfoExtractor):
326 IE_NAME = "crunchyroll
:playlist
"
327 _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
330 'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
332 'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
333 'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
335 'playlist_count': 13,
338 def _real_extract(self, url):
339 show_id = self._match_id(url)
341 webpage = self._download_webpage(url, show_id)
342 title = self._html_search_regex(
343 r'(?s)<h1[^>]*>\s*<span itemprop="name
">(.*?)</span>',
345 episode_paths = re.findall(
346 r'(?s)<li id="showview_videos_media_
[0-9]+"[^>]+>.*?<a href="([^
"]+)"',
349 self.url_result('http
://www
.crunchyroll
.com
' + ep, 'Crunchyroll
')
350 for ep in episode_paths