]>
Commit | Line | Data |
---|---|---|
c8434e83 | 1 | # encoding: utf-8 |
38a40276 | 2 | from __future__ import unicode_literals |
3 | ||
34440095 | 4 | import re |
1d430674 | 5 | import json |
34440095 S |
6 | import base64 |
7 | import zlib | |
8 | ||
c8434e83 | 9 | from hashlib import sha1 |
10 | from math import pow, sqrt, floor | |
11 | from .common import InfoExtractor | |
12 | from ..utils import ( | |
13 | ExtractorError, | |
14 | compat_urllib_parse, | |
15 | compat_urllib_request, | |
16 | bytes_to_intlist, | |
17 | intlist_to_bytes, | |
18 | unified_strdate, | |
19 | clean_html, | |
20 | ) | |
21 | from ..aes import ( | |
22 | aes_cbc_decrypt, | |
23 | inc, | |
24 | ) | |
25 | ||
34440095 | 26 | |
c8434e83 | 27 | class CrunchyrollIE(InfoExtractor): |
34440095 S |
28 | _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)' |
29 | _TEST = { | |
38a40276 | 30 | 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513', |
38a40276 | 31 | #'md5': 'b1639fd6ddfaa43788c85f6d1dddd412', |
32 | 'info_dict': { | |
34440095 S |
33 | 'id': '645513', |
34 | 'ext': 'flv', | |
38a40276 | 35 | 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!', |
36 | 'description': 'md5:2d17137920c64f2f49981a7797d275ef', | |
37 | 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg', | |
38 | 'uploader': 'Yomiuri Telecasting Corporation (YTV)', | |
39 | 'upload_date': '20131013', | |
c8434e83 | 40 | }, |
38a40276 | 41 | 'params': { |
c8434e83 | 42 | # rtmp |
38a40276 | 43 | 'skip_download': True, |
c8434e83 | 44 | }, |
34440095 | 45 | } |
c8434e83 | 46 | |
47 | _FORMAT_IDS = { | |
38a40276 | 48 | '360': ('60', '106'), |
49 | '480': ('61', '106'), | |
50 | '720': ('62', '106'), | |
51 | '1080': ('80', '108'), | |
c8434e83 | 52 | } |
53 | ||
54 | def _decrypt_subtitles(self, data, iv, id): | |
55 | data = bytes_to_intlist(data) | |
56 | iv = bytes_to_intlist(iv) | |
57 | id = int(id) | |
58 | ||
59 | def obfuscate_key_aux(count, modulo, start): | |
60 | output = list(start) | |
61 | for _ in range(count): | |
62 | output.append(output[-1] + output[-2]) | |
63 | # cut off start values | |
64 | output = output[2:] | |
65 | output = list(map(lambda x: x % modulo + 33, output)) | |
66 | return output | |
67 | ||
68 | def obfuscate_key(key): | |
69 | num1 = int(floor(pow(2, 25) * sqrt(6.9))) | |
70 | num2 = (num1 ^ key) << 5 | |
71 | num3 = key ^ num1 | |
72 | num4 = num3 ^ (num3 >> 3) ^ num2 | |
73 | prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2))) | |
38a40276 | 74 | shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode('ascii')).digest()) |
c8434e83 | 75 | # Extend 160 Bit hash to 256 Bit |
76 | return shaHash + [0] * 12 | |
34440095 | 77 | |
c8434e83 | 78 | key = obfuscate_key(id) |
79 | class Counter: | |
80 | __value = iv | |
81 | def next_value(self): | |
82 | temp = self.__value | |
83 | self.__value = inc(self.__value) | |
84 | return temp | |
85 | decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv)) | |
86 | return zlib.decompress(decrypted_data) | |
87 | ||
88 | def _convert_subtitles_to_srt(self, subtitles): | |
38a40276 | 89 | output = '' |
d0a72674 | 90 | for i, (start, end, text) in enumerate(re.findall(r'<event [^>]*?start="([^"]+)" [^>]*?end="([^"]+)" [^>]*?text="([^"]+)"[^>]*?>', subtitles), 1): |
38a40276 | 91 | start = start.replace('.', ',') |
92 | end = end.replace('.', ',') | |
c8434e83 | 93 | text = clean_html(text) |
38a40276 | 94 | text = text.replace('\\N', '\n') |
c8434e83 | 95 | if not text: |
96 | continue | |
38a40276 | 97 | output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text) |
c8434e83 | 98 | return output |
99 | ||
100 | def _real_extract(self,url): | |
101 | mobj = re.match(self._VALID_URL, url) | |
38a40276 | 102 | video_id = mobj.group('video_id') |
103 | ||
104 | if mobj.group('prefix') == 'm': | |
105 | mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage') | |
106 | webpage_url = self._search_regex(r'<link rel="canonical" href="([^"]+)" />', mobile_webpage, 'webpage_url') | |
107 | else: | |
108 | webpage_url = 'http://www.' + mobj.group('url') | |
c8434e83 | 109 | |
38a40276 | 110 | webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage') |
111 | note_m = self._html_search_regex(r'<div class="showmedia-trailer-notice">(.+?)</div>', webpage, 'trailer-notice', default='') | |
c8434e83 | 112 | if note_m: |
113 | raise ExtractorError(note_m) | |
114 | ||
1d430674 S |
115 | mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage) |
116 | if mobj: | |
117 | msg = json.loads(mobj.group('msg')) | |
118 | if msg.get('type') == 'error': | |
119 | raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True) | |
120 | ||
38a40276 | 121 | video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL) |
122 | video_title = re.sub(r' {2,}', ' ', video_title) | |
123 | video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='') | |
c8434e83 | 124 | if not video_description: |
125 | video_description = None | |
38a40276 | 126 | video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL) |
c8434e83 | 127 | if video_upload_date: |
128 | video_upload_date = unified_strdate(video_upload_date) | |
38a40276 | 129 | video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL) |
c8434e83 | 130 | |
38a40276 | 131 | playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url')) |
c8434e83 | 132 | playerdata_req = compat_urllib_request.Request(playerdata_url) |
38a40276 | 133 | playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url}) |
134 | playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded') | |
135 | playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info') | |
34440095 | 136 | |
38a40276 | 137 | stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id') |
138 | video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False) | |
c8434e83 | 139 | |
140 | formats = [] | |
141 | for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage): | |
142 | stream_quality, stream_format = self._FORMAT_IDS[fmt] | |
38a40276 | 143 | video_format = fmt+'p' |
144 | streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/') | |
c8434e83 | 145 | # urlencode doesn't work! |
38a40276 | 146 | streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality='+stream_quality+'&media%5Fid='+stream_id+'&video%5Fformat='+stream_format |
147 | streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded') | |
148 | streamdata_req.add_header('Content-Length', str(len(streamdata_req.data))) | |
149 | streamdata = self._download_webpage(streamdata_req, video_id, note='Downloading media info for '+video_format) | |
150 | video_url = self._search_regex(r'<host>([^<]+)', streamdata, 'video_url') | |
151 | video_play_path = self._search_regex(r'<file>([^<]+)', streamdata, 'video_play_path') | |
c8434e83 | 152 | formats.append({ |
38a40276 | 153 | 'url': video_url, |
154 | 'play_path': video_play_path, | |
155 | 'ext': 'flv', | |
156 | 'format': video_format, | |
157 | 'format_id': video_format, | |
c8434e83 | 158 | }) |
159 | ||
160 | subtitles = {} | |
161 | for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): | |
38a40276 | 162 | sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\ |
163 | video_id, note='Downloading subtitles for '+sub_name) | |
164 | id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False) | |
165 | iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False) | |
166 | data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False) | |
c8434e83 | 167 | if not id or not iv or not data: |
168 | continue | |
169 | id = int(id) | |
170 | iv = base64.b64decode(iv) | |
171 | data = base64.b64decode(data) | |
172 | ||
38a40276 | 173 | subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8') |
70cb7392 | 174 | lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False) |
c8434e83 | 175 | if not lang_code: |
176 | continue | |
177 | subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle) | |
178 | ||
179 | return { | |
38a40276 | 180 | 'id': video_id, |
181 | 'title': video_title, | |
182 | 'description': video_description, | |
183 | 'thumbnail': video_thumbnail, | |
184 | 'uploader': video_uploader, | |
185 | 'upload_date': video_upload_date, | |
186 | 'subtitles': subtitles, | |
187 | 'formats': formats, | |
d0a72674 | 188 | } |