]>
Commit | Line | Data |
---|---|---|
c8434e83 | 1 | # encoding: utf-8 |
38a40276 | 2 | from __future__ import unicode_literals |
3 | ||
34440095 | 4 | import re |
1d430674 | 5 | import json |
34440095 S |
6 | import base64 |
7 | import zlib | |
78272a07 | 8 | import xml.etree.ElementTree |
34440095 | 9 | |
c8434e83 | 10 | from hashlib import sha1 |
11 | from math import pow, sqrt, floor | |
b5857f62 | 12 | from .common import InfoExtractor |
1cc79574 | 13 | from ..compat import ( |
c8434e83 | 14 | compat_urllib_parse, |
a60cccbf | 15 | compat_urllib_parse_unquote, |
c8434e83 | 16 | compat_urllib_request, |
a01da8bb | 17 | compat_urlparse, |
1cc79574 PH |
18 | ) |
19 | from ..utils import ( | |
20 | ExtractorError, | |
c8434e83 | 21 | bytes_to_intlist, |
22 | intlist_to_bytes, | |
725d1c58 | 23 | int_or_none, |
a01da8bb | 24 | remove_end, |
c8434e83 | 25 | unified_strdate, |
723e04d0 | 26 | urlencode_postdata, |
725d1c58 | 27 | xpath_text, |
c8434e83 | 28 | ) |
29 | from ..aes import ( | |
30 | aes_cbc_decrypt, | |
c8434e83 | 31 | ) |
32 | ||
34440095 | 33 | |
12810c9c S |
34 | class CrunchyrollBaseIE(InfoExtractor): |
35 | def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None): | |
36 | request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request) | |
37 | else compat_urllib_request.Request(url_or_request)) | |
38 | # Accept-Language must be set explicitly to accept any language to avoid issues | |
39 | # similar to https://github.com/rg3/youtube-dl/issues/6797. | |
40 | # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction | |
41 | # should be imposed or not (from what I can see it just takes the first language | |
42 | # ignoring the priority and requires it to correspond the IP). By the way this causes | |
43 | # Crunchyroll to not work in georestriction cases in some browsers that don't place | |
44 | # the locale lang first in header. However allowing any language seems to workaround the issue. | |
45 | request.add_header('Accept-Language', '*') | |
46 | return super(CrunchyrollBaseIE, self)._download_webpage( | |
47 | request, video_id, note, errnote, fatal, tries, timeout, encoding) | |
48 | ||
49 | ||
50 | class CrunchyrollIE(CrunchyrollBaseIE): | |
ede21449 | 51 | _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)' |
499bfcbf | 52 | _NETRC_MACHINE = 'crunchyroll' |
a8896c5a | 53 | _TESTS = [{ |
38a40276 | 54 | 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513', |
38a40276 | 55 | 'info_dict': { |
34440095 S |
56 | 'id': '645513', |
57 | 'ext': 'flv', | |
38a40276 | 58 | 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!', |
59 | 'description': 'md5:2d17137920c64f2f49981a7797d275ef', | |
60 | 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg', | |
61 | 'uploader': 'Yomiuri Telecasting Corporation (YTV)', | |
62 | 'upload_date': '20131013', | |
b1edd7a4 | 63 | 'url': 're:(?!.*&)', |
c8434e83 | 64 | }, |
38a40276 | 65 | 'params': { |
c8434e83 | 66 | # rtmp |
38a40276 | 67 | 'skip_download': True, |
c8434e83 | 68 | }, |
ede21449 S |
69 | }, { |
70 | 'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1', | |
71 | 'info_dict': { | |
72 | 'id': '589804', | |
73 | 'ext': 'flv', | |
74 | 'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11', | |
75 | 'description': 'md5:fe2743efedb49d279552926d0bd0cd9e', | |
76 | 'thumbnail': 're:^https?://.*\.jpg$', | |
77 | 'uploader': 'Danny Choo Network', | |
78 | 'upload_date': '20120213', | |
79 | }, | |
80 | 'params': { | |
81 | # rtmp | |
82 | 'skip_download': True, | |
83 | }, | |
84 | ||
a8896c5a S |
85 | }, { |
86 | 'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697', | |
87 | 'only_matching': True, | |
88 | }] | |
c8434e83 | 89 | |
90 | _FORMAT_IDS = { | |
38a40276 | 91 | '360': ('60', '106'), |
92 | '480': ('61', '106'), | |
93 | '720': ('62', '106'), | |
94 | '1080': ('80', '108'), | |
c8434e83 | 95 | } |
96 | ||
723e04d0 A |
97 | def _login(self): |
98 | (username, password) = self._get_login_info() | |
99 | if username is None: | |
100 | return | |
101 | self.report_login() | |
102 | login_url = 'https://www.crunchyroll.com/?a=formhandler' | |
103 | data = urlencode_postdata({ | |
104 | 'formname': 'RpcApiUser_Login', | |
105 | 'name': username, | |
106 | 'password': password, | |
107 | }) | |
108 | login_request = compat_urllib_request.Request(login_url, data) | |
109 | login_request.add_header('Content-Type', 'application/x-www-form-urlencoded') | |
110 | self._download_webpage(login_request, None, False, 'Wrong login info') | |
111 | ||
112 | def _real_initialize(self): | |
113 | self._login() | |
114 | ||
c8434e83 | 115 | def _decrypt_subtitles(self, data, iv, id): |
1a5b77dc S |
116 | data = bytes_to_intlist(base64.b64decode(data.encode('utf-8'))) |
117 | iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8'))) | |
c8434e83 | 118 | id = int(id) |
119 | ||
120 | def obfuscate_key_aux(count, modulo, start): | |
121 | output = list(start) | |
122 | for _ in range(count): | |
123 | output.append(output[-1] + output[-2]) | |
124 | # cut off start values | |
125 | output = output[2:] | |
126 | output = list(map(lambda x: x % modulo + 33, output)) | |
127 | return output | |
128 | ||
129 | def obfuscate_key(key): | |
130 | num1 = int(floor(pow(2, 25) * sqrt(6.9))) | |
131 | num2 = (num1 ^ key) << 5 | |
132 | num3 = key ^ num1 | |
133 | num4 = num3 ^ (num3 >> 3) ^ num2 | |
134 | prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2))) | |
38a40276 | 135 | shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode('ascii')).digest()) |
c8434e83 | 136 | # Extend 160 Bit hash to 256 Bit |
137 | return shaHash + [0] * 12 | |
34440095 | 138 | |
c8434e83 | 139 | key = obfuscate_key(id) |
5f6a1245 | 140 | |
c8434e83 | 141 | decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv)) |
142 | return zlib.decompress(decrypted_data) | |
143 | ||
d65d6286 | 144 | def _convert_subtitles_to_srt(self, sub_root): |
38a40276 | 145 | output = '' |
d65d6286 JMF |
146 | |
147 | for i, event in enumerate(sub_root.findall('./events/event'), 1): | |
148 | start = event.attrib['start'].replace('.', ',') | |
149 | end = event.attrib['end'].replace('.', ',') | |
150 | text = event.attrib['text'].replace('\\N', '\n') | |
38a40276 | 151 | output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text) |
c8434e83 | 152 | return output |
153 | ||
d65d6286 | 154 | def _convert_subtitles_to_ass(self, sub_root): |
78272a07 A |
155 | output = '' |
156 | ||
157 | def ass_bool(strvalue): | |
158 | assvalue = '0' | |
159 | if strvalue == '1': | |
160 | assvalue = '-1' | |
161 | return assvalue | |
162 | ||
78272a07 A |
163 | output = '[Script Info]\n' |
164 | output += 'Title: %s\n' % sub_root.attrib["title"] | |
165 | output += 'ScriptType: v4.00+\n' | |
166 | output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"] | |
167 | output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"] | |
168 | output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"] | |
169 | output += """ScaledBorderAndShadow: yes | |
170 | ||
171 | [V4+ Styles] | |
172 | Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding | |
173 | """ | |
174 | for style in sub_root.findall('./styles/style'): | |
175 | output += 'Style: ' + style.attrib["name"] | |
176 | output += ',' + style.attrib["font_name"] | |
177 | output += ',' + style.attrib["font_size"] | |
178 | output += ',' + style.attrib["primary_colour"] | |
179 | output += ',' + style.attrib["secondary_colour"] | |
180 | output += ',' + style.attrib["outline_colour"] | |
181 | output += ',' + style.attrib["back_colour"] | |
182 | output += ',' + ass_bool(style.attrib["bold"]) | |
183 | output += ',' + ass_bool(style.attrib["italic"]) | |
184 | output += ',' + ass_bool(style.attrib["underline"]) | |
185 | output += ',' + ass_bool(style.attrib["strikeout"]) | |
186 | output += ',' + style.attrib["scale_x"] | |
187 | output += ',' + style.attrib["scale_y"] | |
188 | output += ',' + style.attrib["spacing"] | |
189 | output += ',' + style.attrib["angle"] | |
190 | output += ',' + style.attrib["border_style"] | |
191 | output += ',' + style.attrib["outline"] | |
192 | output += ',' + style.attrib["shadow"] | |
193 | output += ',' + style.attrib["alignment"] | |
194 | output += ',' + style.attrib["margin_l"] | |
195 | output += ',' + style.attrib["margin_r"] | |
196 | output += ',' + style.attrib["margin_v"] | |
197 | output += ',' + style.attrib["encoding"] | |
198 | output += '\n' | |
199 | ||
200 | output += """ | |
201 | [Events] | |
202 | Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text | |
203 | """ | |
204 | for event in sub_root.findall('./events/event'): | |
205 | output += 'Dialogue: 0' | |
206 | output += ',' + event.attrib["start"] | |
207 | output += ',' + event.attrib["end"] | |
208 | output += ',' + event.attrib["style"] | |
209 | output += ',' + event.attrib["name"] | |
210 | output += ',' + event.attrib["margin_l"] | |
211 | output += ',' + event.attrib["margin_r"] | |
212 | output += ',' + event.attrib["margin_v"] | |
213 | output += ',' + event.attrib["effect"] | |
214 | output += ',' + event.attrib["text"] | |
215 | output += '\n' | |
216 | ||
217 | return output | |
218 | ||
0385d642 S |
219 | def _extract_subtitles(self, subtitle): |
220 | sub_root = xml.etree.ElementTree.fromstring(subtitle) | |
221 | return [{ | |
222 | 'ext': 'srt', | |
223 | 'data': self._convert_subtitles_to_srt(sub_root), | |
224 | }, { | |
225 | 'ext': 'ass', | |
226 | 'data': self._convert_subtitles_to_ass(sub_root), | |
227 | }] | |
228 | ||
b5857f62 JMF |
229 | def _get_subtitles(self, video_id, webpage): |
230 | subtitles = {} | |
231 | for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): | |
232 | sub_page = self._download_webpage( | |
233 | 'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id, | |
234 | video_id, note='Downloading subtitles for ' + sub_name) | |
235 | id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False) | |
236 | iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False) | |
237 | data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False) | |
238 | if not id or not iv or not data: | |
239 | continue | |
b5857f62 JMF |
240 | subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8') |
241 | lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False) | |
242 | if not lang_code: | |
243 | continue | |
0385d642 | 244 | subtitles[lang_code] = self._extract_subtitles(subtitle) |
b5857f62 JMF |
245 | return subtitles |
246 | ||
5f6a1245 | 247 | def _real_extract(self, url): |
c8434e83 | 248 | mobj = re.match(self._VALID_URL, url) |
38a40276 | 249 | video_id = mobj.group('video_id') |
250 | ||
251 | if mobj.group('prefix') == 'm': | |
252 | mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage') | |
253 | webpage_url = self._search_regex(r'<link rel="canonical" href="([^"]+)" />', mobile_webpage, 'webpage_url') | |
254 | else: | |
255 | webpage_url = 'http://www.' + mobj.group('url') | |
c8434e83 | 256 | |
38a40276 | 257 | webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage') |
2f72e83b S |
258 | note_m = self._html_search_regex( |
259 | r'<div class="showmedia-trailer-notice">(.+?)</div>', | |
260 | webpage, 'trailer-notice', default='') | |
c8434e83 | 261 | if note_m: |
262 | raise ExtractorError(note_m) | |
263 | ||
1d430674 S |
264 | mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage) |
265 | if mobj: | |
266 | msg = json.loads(mobj.group('msg')) | |
267 | if msg.get('type') == 'error': | |
268 | raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True) | |
2f72e83b S |
269 | |
270 | if 'To view this, please log in to verify you are 18 or older.' in webpage: | |
39affb5a | 271 | self.raise_login_required() |
1d430674 | 272 | |
38a40276 | 273 | video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL) |
274 | video_title = re.sub(r' {2,}', ' ', video_title) | |
275 | video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='') | |
c8434e83 | 276 | if not video_description: |
277 | video_description = None | |
38a40276 | 278 | video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL) |
c8434e83 | 279 | if video_upload_date: |
280 | video_upload_date = unified_strdate(video_upload_date) | |
38a40276 | 281 | video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL) |
c8434e83 | 282 | |
a60cccbf | 283 | playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url')) |
c8434e83 | 284 | playerdata_req = compat_urllib_request.Request(playerdata_url) |
38a40276 | 285 | playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url}) |
286 | playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded') | |
287 | playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info') | |
34440095 | 288 | |
38a40276 | 289 | stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id') |
290 | video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False) | |
c8434e83 | 291 | |
292 | formats = [] | |
a221f229 | 293 | for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage): |
c8434e83 | 294 | stream_quality, stream_format = self._FORMAT_IDS[fmt] |
2514d263 | 295 | video_format = fmt + 'p' |
ede21449 S |
296 | streamdata_req = compat_urllib_request.Request( |
297 | 'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s' | |
298 | % (stream_id, stream_format, stream_quality), | |
299 | compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8')) | |
38a40276 | 300 | streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded') |
b1edd7a4 PH |
301 | streamdata = self._download_xml( |
302 | streamdata_req, video_id, | |
303 | note='Downloading media info for %s' % video_format) | |
ede21449 S |
304 | stream_info = streamdata.find('./{default}preload/stream_info') |
305 | video_url = stream_info.find('./host').text | |
306 | video_play_path = stream_info.find('./file').text | |
725d1c58 JMF |
307 | metadata = stream_info.find('./metadata') |
308 | format_info = { | |
309 | 'format': video_format, | |
310 | 'format_id': video_format, | |
311 | 'height': int_or_none(xpath_text(metadata, './height')), | |
312 | 'width': int_or_none(xpath_text(metadata, './width')), | |
313 | } | |
a01da8bb S |
314 | |
315 | if '.fplive.net/' in video_url: | |
316 | video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip()) | |
317 | parsed_video_url = compat_urlparse.urlparse(video_url) | |
318 | direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace( | |
319 | netloc='v.lvlt.crcdn.net', | |
320 | path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_play_path.split(':')[-1]))) | |
321 | if self._is_valid_url(direct_video_url, video_id, video_format): | |
725d1c58 | 322 | format_info.update({ |
a01da8bb | 323 | 'url': direct_video_url, |
a01da8bb | 324 | }) |
725d1c58 | 325 | formats.append(format_info) |
a01da8bb S |
326 | continue |
327 | ||
725d1c58 | 328 | format_info.update({ |
38a40276 | 329 | 'url': video_url, |
b1edd7a4 | 330 | 'play_path': video_play_path, |
38a40276 | 331 | 'ext': 'flv', |
c8434e83 | 332 | }) |
725d1c58 | 333 | formats.append(format_info) |
c8434e83 | 334 | |
b5857f62 | 335 | subtitles = self.extract_subtitles(video_id, webpage) |
11b3ce85 | 336 | |
c8434e83 | 337 | return { |
8bcc8756 JW |
338 | 'id': video_id, |
339 | 'title': video_title, | |
38a40276 | 340 | 'description': video_description, |
8bcc8756 JW |
341 | 'thumbnail': video_thumbnail, |
342 | 'uploader': video_uploader, | |
38a40276 | 343 | 'upload_date': video_upload_date, |
8bcc8756 JW |
344 | 'subtitles': subtitles, |
345 | 'formats': formats, | |
d0a72674 | 346 | } |
8230018c GS |
347 | |
348 | ||
12810c9c | 349 | class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE): |
8230018c | 350 | IE_NAME = "crunchyroll:playlist" |
09e5d6a6 | 351 | _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$' |
8230018c GS |
352 | |
353 | _TESTS = [{ | |
09e5d6a6 PH |
354 | 'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi', |
355 | 'info_dict': { | |
356 | 'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi', | |
357 | 'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi' | |
8230018c | 358 | }, |
09e5d6a6 | 359 | 'playlist_count': 13, |
8230018c GS |
360 | }] |
361 | ||
8230018c | 362 | def _real_extract(self, url): |
09e5d6a6 PH |
363 | show_id = self._match_id(url) |
364 | ||
365 | webpage = self._download_webpage(url, show_id) | |
366 | title = self._html_search_regex( | |
367 | r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>', | |
368 | webpage, 'title') | |
369 | episode_paths = re.findall( | |
370 | r'(?s)<li id="showview_videos_media_[0-9]+"[^>]+>.*?<a href="([^"]+)"', | |
371 | webpage) | |
372 | entries = [ | |
373 | self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll') | |
374 | for ep in episode_paths | |
375 | ] | |
376 | entries.reverse() | |
377 | ||
8230018c | 378 | return { |
09e5d6a6 PH |
379 | '_type': 'playlist', |
380 | 'id': show_id, | |
381 | 'title': title, | |
382 | 'entries': entries, | |
383 | } |