]>
Commit | Line | Data |
---|---|---|
c8434e83 | 1 | # encoding: utf-8 |
38a40276 | 2 | from __future__ import unicode_literals |
3 | ||
34440095 | 4 | import re |
1d430674 | 5 | import json |
34440095 S |
6 | import base64 |
7 | import zlib | |
78272a07 | 8 | import xml.etree.ElementTree |
34440095 | 9 | |
c8434e83 | 10 | from hashlib import sha1 |
11 | from math import pow, sqrt, floor | |
11b3ce85 | 12 | from .subtitles import SubtitlesInfoExtractor |
c8434e83 | 13 | from ..utils import ( |
14 | ExtractorError, | |
15 | compat_urllib_parse, | |
16 | compat_urllib_request, | |
17 | bytes_to_intlist, | |
18 | intlist_to_bytes, | |
19 | unified_strdate, | |
723e04d0 | 20 | urlencode_postdata, |
c8434e83 | 21 | ) |
22 | from ..aes import ( | |
23 | aes_cbc_decrypt, | |
24 | inc, | |
25 | ) | |
8230018c | 26 | from .common import InfoExtractor |
c8434e83 | 27 | |
34440095 | 28 | |
11b3ce85 | 29 | class CrunchyrollIE(SubtitlesInfoExtractor): |
34440095 S |
30 | _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)' |
31 | _TEST = { | |
38a40276 | 32 | 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513', |
38a40276 | 33 | #'md5': 'b1639fd6ddfaa43788c85f6d1dddd412', |
34 | 'info_dict': { | |
34440095 S |
35 | 'id': '645513', |
36 | 'ext': 'flv', | |
38a40276 | 37 | 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!', |
38 | 'description': 'md5:2d17137920c64f2f49981a7797d275ef', | |
39 | 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg', | |
40 | 'uploader': 'Yomiuri Telecasting Corporation (YTV)', | |
41 | 'upload_date': '20131013', | |
b1edd7a4 | 42 | 'url': 're:(?!.*&)', |
c8434e83 | 43 | }, |
38a40276 | 44 | 'params': { |
c8434e83 | 45 | # rtmp |
38a40276 | 46 | 'skip_download': True, |
c8434e83 | 47 | }, |
34440095 | 48 | } |
c8434e83 | 49 | |
50 | _FORMAT_IDS = { | |
38a40276 | 51 | '360': ('60', '106'), |
52 | '480': ('61', '106'), | |
53 | '720': ('62', '106'), | |
54 | '1080': ('80', '108'), | |
c8434e83 | 55 | } |
56 | ||
723e04d0 A |
57 | def _login(self): |
58 | (username, password) = self._get_login_info() | |
59 | if username is None: | |
60 | return | |
61 | self.report_login() | |
62 | login_url = 'https://www.crunchyroll.com/?a=formhandler' | |
63 | data = urlencode_postdata({ | |
64 | 'formname': 'RpcApiUser_Login', | |
65 | 'name': username, | |
66 | 'password': password, | |
67 | }) | |
68 | login_request = compat_urllib_request.Request(login_url, data) | |
69 | login_request.add_header('Content-Type', 'application/x-www-form-urlencoded') | |
70 | self._download_webpage(login_request, None, False, 'Wrong login info') | |
71 | ||
78272a07 | 72 | |
723e04d0 A |
73 | def _real_initialize(self): |
74 | self._login() | |
75 | ||
78272a07 | 76 | |
c8434e83 | 77 | def _decrypt_subtitles(self, data, iv, id): |
78 | data = bytes_to_intlist(data) | |
79 | iv = bytes_to_intlist(iv) | |
80 | id = int(id) | |
81 | ||
82 | def obfuscate_key_aux(count, modulo, start): | |
83 | output = list(start) | |
84 | for _ in range(count): | |
85 | output.append(output[-1] + output[-2]) | |
86 | # cut off start values | |
87 | output = output[2:] | |
88 | output = list(map(lambda x: x % modulo + 33, output)) | |
89 | return output | |
90 | ||
91 | def obfuscate_key(key): | |
92 | num1 = int(floor(pow(2, 25) * sqrt(6.9))) | |
93 | num2 = (num1 ^ key) << 5 | |
94 | num3 = key ^ num1 | |
95 | num4 = num3 ^ (num3 >> 3) ^ num2 | |
96 | prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2))) | |
38a40276 | 97 | shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode('ascii')).digest()) |
c8434e83 | 98 | # Extend 160 Bit hash to 256 Bit |
99 | return shaHash + [0] * 12 | |
34440095 | 100 | |
c8434e83 | 101 | key = obfuscate_key(id) |
102 | class Counter: | |
103 | __value = iv | |
104 | def next_value(self): | |
105 | temp = self.__value | |
106 | self.__value = inc(self.__value) | |
107 | return temp | |
108 | decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv)) | |
109 | return zlib.decompress(decrypted_data) | |
110 | ||
d65d6286 | 111 | def _convert_subtitles_to_srt(self, sub_root): |
38a40276 | 112 | output = '' |
d65d6286 JMF |
113 | |
114 | for i, event in enumerate(sub_root.findall('./events/event'), 1): | |
115 | start = event.attrib['start'].replace('.', ',') | |
116 | end = event.attrib['end'].replace('.', ',') | |
117 | text = event.attrib['text'].replace('\\N', '\n') | |
38a40276 | 118 | output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text) |
c8434e83 | 119 | return output |
120 | ||
d65d6286 | 121 | def _convert_subtitles_to_ass(self, sub_root): |
78272a07 A |
122 | output = '' |
123 | ||
124 | def ass_bool(strvalue): | |
125 | assvalue = '0' | |
126 | if strvalue == '1': | |
127 | assvalue = '-1' | |
128 | return assvalue | |
129 | ||
78272a07 A |
130 | output = '[Script Info]\n' |
131 | output += 'Title: %s\n' % sub_root.attrib["title"] | |
132 | output += 'ScriptType: v4.00+\n' | |
133 | output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"] | |
134 | output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"] | |
135 | output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"] | |
136 | output += """ScaledBorderAndShadow: yes | |
137 | ||
138 | [V4+ Styles] | |
139 | Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding | |
140 | """ | |
141 | for style in sub_root.findall('./styles/style'): | |
142 | output += 'Style: ' + style.attrib["name"] | |
143 | output += ',' + style.attrib["font_name"] | |
144 | output += ',' + style.attrib["font_size"] | |
145 | output += ',' + style.attrib["primary_colour"] | |
146 | output += ',' + style.attrib["secondary_colour"] | |
147 | output += ',' + style.attrib["outline_colour"] | |
148 | output += ',' + style.attrib["back_colour"] | |
149 | output += ',' + ass_bool(style.attrib["bold"]) | |
150 | output += ',' + ass_bool(style.attrib["italic"]) | |
151 | output += ',' + ass_bool(style.attrib["underline"]) | |
152 | output += ',' + ass_bool(style.attrib["strikeout"]) | |
153 | output += ',' + style.attrib["scale_x"] | |
154 | output += ',' + style.attrib["scale_y"] | |
155 | output += ',' + style.attrib["spacing"] | |
156 | output += ',' + style.attrib["angle"] | |
157 | output += ',' + style.attrib["border_style"] | |
158 | output += ',' + style.attrib["outline"] | |
159 | output += ',' + style.attrib["shadow"] | |
160 | output += ',' + style.attrib["alignment"] | |
161 | output += ',' + style.attrib["margin_l"] | |
162 | output += ',' + style.attrib["margin_r"] | |
163 | output += ',' + style.attrib["margin_v"] | |
164 | output += ',' + style.attrib["encoding"] | |
165 | output += '\n' | |
166 | ||
167 | output += """ | |
168 | [Events] | |
169 | Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text | |
170 | """ | |
171 | for event in sub_root.findall('./events/event'): | |
172 | output += 'Dialogue: 0' | |
173 | output += ',' + event.attrib["start"] | |
174 | output += ',' + event.attrib["end"] | |
175 | output += ',' + event.attrib["style"] | |
176 | output += ',' + event.attrib["name"] | |
177 | output += ',' + event.attrib["margin_l"] | |
178 | output += ',' + event.attrib["margin_r"] | |
179 | output += ',' + event.attrib["margin_v"] | |
180 | output += ',' + event.attrib["effect"] | |
181 | output += ',' + event.attrib["text"] | |
182 | output += '\n' | |
183 | ||
184 | return output | |
185 | ||
c8434e83 | 186 | def _real_extract(self,url): |
187 | mobj = re.match(self._VALID_URL, url) | |
38a40276 | 188 | video_id = mobj.group('video_id') |
189 | ||
190 | if mobj.group('prefix') == 'm': | |
191 | mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage') | |
192 | webpage_url = self._search_regex(r'<link rel="canonical" href="([^"]+)" />', mobile_webpage, 'webpage_url') | |
193 | else: | |
194 | webpage_url = 'http://www.' + mobj.group('url') | |
c8434e83 | 195 | |
38a40276 | 196 | webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage') |
197 | note_m = self._html_search_regex(r'<div class="showmedia-trailer-notice">(.+?)</div>', webpage, 'trailer-notice', default='') | |
c8434e83 | 198 | if note_m: |
199 | raise ExtractorError(note_m) | |
200 | ||
1d430674 S |
201 | mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage) |
202 | if mobj: | |
203 | msg = json.loads(mobj.group('msg')) | |
204 | if msg.get('type') == 'error': | |
205 | raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True) | |
206 | ||
38a40276 | 207 | video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL) |
208 | video_title = re.sub(r' {2,}', ' ', video_title) | |
209 | video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='') | |
c8434e83 | 210 | if not video_description: |
211 | video_description = None | |
38a40276 | 212 | video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL) |
c8434e83 | 213 | if video_upload_date: |
214 | video_upload_date = unified_strdate(video_upload_date) | |
38a40276 | 215 | video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL) |
c8434e83 | 216 | |
38a40276 | 217 | playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url')) |
c8434e83 | 218 | playerdata_req = compat_urllib_request.Request(playerdata_url) |
38a40276 | 219 | playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url}) |
220 | playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded') | |
221 | playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info') | |
34440095 | 222 | |
38a40276 | 223 | stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id') |
224 | video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False) | |
c8434e83 | 225 | |
226 | formats = [] | |
227 | for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage): | |
228 | stream_quality, stream_format = self._FORMAT_IDS[fmt] | |
38a40276 | 229 | video_format = fmt+'p' |
230 | streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/') | |
c8434e83 | 231 | # urlencode doesn't work! |
38a40276 | 232 | streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality='+stream_quality+'&media%5Fid='+stream_id+'&video%5Fformat='+stream_format |
233 | streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded') | |
234 | streamdata_req.add_header('Content-Length', str(len(streamdata_req.data))) | |
b1edd7a4 PH |
235 | streamdata = self._download_xml( |
236 | streamdata_req, video_id, | |
237 | note='Downloading media info for %s' % video_format) | |
238 | video_url = streamdata.find('.//host').text | |
239 | video_play_path = streamdata.find('.//file').text | |
c8434e83 | 240 | formats.append({ |
38a40276 | 241 | 'url': video_url, |
b1edd7a4 | 242 | 'play_path': video_play_path, |
38a40276 | 243 | 'ext': 'flv', |
244 | 'format': video_format, | |
245 | 'format_id': video_format, | |
c8434e83 | 246 | }) |
247 | ||
248 | subtitles = {} | |
78272a07 | 249 | sub_format = self._downloader.params.get('subtitlesformat', 'srt') |
c8434e83 | 250 | for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): |
38a40276 | 251 | sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\ |
252 | video_id, note='Downloading subtitles for '+sub_name) | |
253 | id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False) | |
254 | iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False) | |
255 | data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False) | |
c8434e83 | 256 | if not id or not iv or not data: |
257 | continue | |
258 | id = int(id) | |
259 | iv = base64.b64decode(iv) | |
260 | data = base64.b64decode(data) | |
261 | ||
38a40276 | 262 | subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8') |
70cb7392 | 263 | lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False) |
c8434e83 | 264 | if not lang_code: |
265 | continue | |
d65d6286 | 266 | sub_root = xml.etree.ElementTree.fromstring(subtitle) |
78272a07 | 267 | if sub_format == 'ass': |
d65d6286 | 268 | subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root) |
78272a07 | 269 | else: |
d65d6286 | 270 | subtitles[lang_code] = self._convert_subtitles_to_srt(sub_root) |
c8434e83 | 271 | |
11b3ce85 NJ |
272 | if self._downloader.params.get('listsubtitles', False): |
273 | self._list_available_subtitles(video_id, subtitles) | |
274 | return | |
275 | ||
c8434e83 | 276 | return { |
38a40276 | 277 | 'id': video_id, |
278 | 'title': video_title, | |
279 | 'description': video_description, | |
280 | 'thumbnail': video_thumbnail, | |
281 | 'uploader': video_uploader, | |
282 | 'upload_date': video_upload_date, | |
283 | 'subtitles': subtitles, | |
284 | 'formats': formats, | |
d0a72674 | 285 | } |
8230018c GS |
286 | |
287 | ||
288 | class CrunchyrollShowPlaylistIE(InfoExtractor): | |
289 | IE_NAME = "crunchyroll:playlist" | |
09e5d6a6 | 290 | _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$' |
8230018c GS |
291 | |
292 | _TESTS = [{ | |
09e5d6a6 PH |
293 | 'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi', |
294 | 'info_dict': { | |
295 | 'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi', | |
296 | 'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi' | |
8230018c | 297 | }, |
09e5d6a6 | 298 | 'playlist_count': 13, |
8230018c GS |
299 | }] |
300 | ||
8230018c | 301 | def _real_extract(self, url): |
09e5d6a6 PH |
302 | show_id = self._match_id(url) |
303 | ||
304 | webpage = self._download_webpage(url, show_id) | |
305 | title = self._html_search_regex( | |
306 | r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>', | |
307 | webpage, 'title') | |
308 | episode_paths = re.findall( | |
309 | r'(?s)<li id="showview_videos_media_[0-9]+"[^>]+>.*?<a href="([^"]+)"', | |
310 | webpage) | |
311 | entries = [ | |
312 | self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll') | |
313 | for ep in episode_paths | |
314 | ] | |
315 | entries.reverse() | |
316 | ||
8230018c | 317 | return { |
09e5d6a6 PH |
318 | '_type': 'playlist', |
319 | 'id': show_id, | |
320 | 'title': title, | |
321 | 'entries': entries, | |
322 | } |