]>
Commit | Line | Data |
---|---|---|
ddbd9035 | 1 | # coding: utf-8 |
8a32b82e PH |
2 | from __future__ import unicode_literals |
3 | ||
411c590a | 4 | import itertools |
034caf70 | 5 | import random |
411c590a | 6 | import re |
034caf70 YCH |
7 | import string |
8 | import time | |
9c286cfa PH |
9 | |
10 | from .common import InfoExtractor | |
5c2266df S |
11 | from ..utils import ( |
12 | ExtractorError, | |
d3d4ba7f YCH |
13 | get_element_by_class, |
14 | js_to_json, | |
d16c0121 | 15 | str_or_none, |
d3d4ba7f YCH |
16 | strip_jsonp, |
17 | urljoin, | |
c203be3f | 18 | ) |
1498940b | 19 | |
aed473cc | 20 | |
9c286cfa | 21 | class YoukuIE(InfoExtractor): |
f9355dc9 | 22 | IE_NAME = 'youku' |
246995db | 23 | IE_DESC = '优酷' |
8a32b82e PH |
24 | _VALID_URL = r'''(?x) |
25 | (?: | |
c130f0a3 YCH |
26 | https?://( |
27 | (?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)| | |
28 | video\.tudou\.com/v/)| | |
8a32b82e PH |
29 | youku:) |
30 | (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|) | |
31 | ''' | |
f9355dc9 | 32 | |
ee697992 | 33 | _TESTS = [{ |
c683454e | 34 | # MD5 is unstable |
aed473cc | 35 | 'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html', |
aed473cc | 36 | 'info_dict': { |
59ed87cb | 37 | 'id': 'XMTc1ODE5Njcy', |
aed473cc | 38 | 'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.', |
59ed87cb | 39 | 'ext': 'mp4', |
d16c0121 YCH |
40 | 'duration': 74.73, |
41 | 'thumbnail': r're:^https?://.*', | |
42 | 'uploader': '。躲猫猫、', | |
43 | 'uploader_id': '36017967', | |
44 | 'uploader_url': 'http://i.youku.com/u/UMTQ0MDcxODY4', | |
45 | 'tags': list, | |
aed473cc | 46 | } |
ee697992 YCH |
47 | }, { |
48 | 'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf', | |
49 | 'only_matching': True, | |
f1e66cb2 YCH |
50 | }, { |
51 | 'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html', | |
52 | 'info_dict': { | |
53 | 'id': 'XODgxNjg1Mzk2', | |
59ed87cb | 54 | 'ext': 'mp4', |
f1e66cb2 | 55 | 'title': '武媚娘传奇 85', |
d16c0121 YCH |
56 | 'duration': 1999.61, |
57 | 'thumbnail': r're:^https?://.*', | |
58 | 'uploader': '疯狂豆花', | |
59 | 'uploader_id': '62583473', | |
60 | 'uploader_url': 'http://i.youku.com/u/UMjUwMzMzODky', | |
61 | 'tags': list, | |
f1e66cb2 | 62 | }, |
5228b756 YCH |
63 | }, { |
64 | 'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html', | |
65 | 'info_dict': { | |
66 | 'id': 'XMTI1OTczNDM5Mg', | |
59ed87cb | 67 | 'ext': 'mp4', |
5228b756 | 68 | 'title': '花千骨 04', |
d16c0121 YCH |
69 | 'duration': 2363, |
70 | 'thumbnail': r're:^https?://.*', | |
71 | 'uploader': '放剧场-花千骨', | |
72 | 'uploader_id': '772849359', | |
73 | 'uploader_url': 'http://i.youku.com/u/UMzA5MTM5NzQzNg==', | |
74 | 'tags': list, | |
5228b756 | 75 | }, |
33eae08f P |
76 | }, { |
77 | 'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html', | |
78 | 'note': 'Video protected with password', | |
79 | 'info_dict': { | |
80 | 'id': 'XNjA1NzA2Njgw', | |
59ed87cb | 81 | 'ext': 'mp4', |
5ddc127d | 82 | 'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起', |
d16c0121 YCH |
83 | 'duration': 7264.5, |
84 | 'thumbnail': r're:^https?://.*', | |
85 | 'uploader': 'FoxJin1006', | |
86 | 'uploader_id': '322014285', | |
87 | 'uploader_url': 'http://i.youku.com/u/UMTI4ODA1NzE0MA==', | |
88 | 'tags': list, | |
33eae08f P |
89 | }, |
90 | 'params': { | |
91 | 'videopassword': '100600', | |
92 | }, | |
eb01e97e YCH |
93 | }, { |
94 | # /play/get.json contains streams with "channel_type":"tail" | |
95 | 'url': 'http://v.youku.com/v_show/id_XOTUxMzg4NDMy.html', | |
96 | 'info_dict': { | |
97 | 'id': 'XOTUxMzg4NDMy', | |
59ed87cb | 98 | 'ext': 'mp4', |
eb01e97e | 99 | 'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft', |
d16c0121 YCH |
100 | 'duration': 702.08, |
101 | 'thumbnail': r're:^https?://.*', | |
102 | 'uploader': '明月庄主moon', | |
103 | 'uploader_id': '38465621', | |
104 | 'uploader_url': 'http://i.youku.com/u/UMTUzODYyNDg0', | |
105 | 'tags': list, | |
eb01e97e | 106 | }, |
c130f0a3 YCH |
107 | }, { |
108 | 'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805', | |
109 | 'info_dict': { | |
110 | 'id': 'XMjIyNzAzMTQ4NA', | |
111 | 'ext': 'mp4', | |
112 | 'title': '卡马乔国足开大脚长传冲吊集锦', | |
d16c0121 YCH |
113 | 'duration': 289, |
114 | 'thumbnail': r're:^https?://.*', | |
115 | 'uploader': '阿卜杜拉之星', | |
116 | 'uploader_id': '2382249', | |
117 | 'uploader_url': 'http://i.youku.com/u/UOTUyODk5Ng==', | |
118 | 'tags': list, | |
c130f0a3 YCH |
119 | }, |
120 | }, { | |
121 | 'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html', | |
122 | 'only_matching': True, | |
ee697992 | 123 | }] |
67f51b3d | 124 | |
034caf70 YCH |
125 | @staticmethod |
126 | def get_ysuid(): | |
127 | return '%d%s' % (int(time.time()), ''.join([ | |
128 | random.choice(string.ascii_letters) for i in range(3)])) | |
129 | ||
08f7db20 P |
130 | def get_format_name(self, fm): |
131 | _dict = { | |
aed473cc YCH |
132 | '3gp': 'h6', |
133 | '3gphd': 'h5', | |
134 | 'flv': 'h4', | |
dbb7d7e2 | 135 | 'flvhd': 'h4', |
aed473cc | 136 | 'mp4': 'h3', |
8696a7fd | 137 | 'mp4hd': 'h3', |
dbb7d7e2 | 138 | 'mp4hd2': 'h4', |
8696a7fd | 139 | 'mp4hd3': 'h4', |
dbb7d7e2 YCH |
140 | 'hd2': 'h2', |
141 | 'hd3': 'h1', | |
08f7db20 | 142 | } |
59ed87cb | 143 | return _dict.get(fm) |
08f7db20 | 144 | |
9c286cfa | 145 | def _real_extract(self, url): |
9383e66f | 146 | video_id = self._match_id(url) |
9c286cfa | 147 | |
034caf70 | 148 | self._set_cookie('youku.com', '__ysuid', self.get_ysuid()) |
59ed87cb | 149 | self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com') |
034caf70 | 150 | |
59ed87cb YCH |
151 | _, urlh = self._download_webpage_handle( |
152 | 'https://log.mmstat.com/eg.js', video_id, 'Retrieving cna info') | |
153 | # The etag header is '"foobar"'; let's remove the double quotes | |
154 | cna = urlh.headers['etag'][1:-1] | |
51094b1b | 155 | |
59ed87cb YCH |
156 | # request basic data |
157 | basic_data_params = { | |
158 | 'vid': video_id, | |
c130f0a3 | 159 | 'ccode': '0402' if 'tudou.com' in url else '0401', |
59ed87cb YCH |
160 | 'client_ip': '192.168.1.1', |
161 | 'utid': cna, | |
162 | 'client_ts': time.time() / 1000, | |
163 | } | |
51094b1b | 164 | |
d800609c | 165 | video_password = self._downloader.params.get('videopassword') |
33eae08f | 166 | if video_password: |
59ed87cb | 167 | basic_data_params['password'] = video_password |
cb3d2eb9 | 168 | |
59ed87cb YCH |
169 | headers = { |
170 | 'Referer': url, | |
171 | } | |
172 | headers.update(self.geo_verification_headers()) | |
173 | data = self._download_json( | |
174 | 'https://ups.youku.com/ups/get.json', video_id, | |
175 | 'Downloading JSON metadata', | |
176 | query=basic_data_params, headers=headers)['data'] | |
8a32b82e | 177 | |
7e37c394 | 178 | error = data.get('error') |
14c17caf C |
179 | if error: |
180 | error_note = error.get('note') | |
181 | if error_note is not None and '因版权原因无法观看此视频' in error_note: | |
04e75966 YCH |
182 | raise ExtractorError( |
183 | 'Youku said: Sorry, this video is available in China only', expected=True) | |
cc799437 S |
184 | elif error_note and '该视频被设为私密' in error_note: |
185 | raise ExtractorError( | |
186 | 'Youku said: Sorry, this video is private', expected=True) | |
04e75966 | 187 | else: |
14c17caf | 188 | msg = 'Youku server reported error %i' % error.get('code') |
35e22b6b | 189 | if error_note is not None: |
14c17caf | 190 | msg += ': ' + error_note |
04e75966 | 191 | raise ExtractorError(msg) |
f9355dc9 | 192 | |
f133fd32 | 193 | # get video title |
d16c0121 YCH |
194 | video_data = data['video'] |
195 | title = video_data['title'] | |
f9355dc9 | 196 | |
59ed87cb YCH |
197 | formats = [{ |
198 | 'url': stream['m3u8_url'], | |
199 | 'format_id': self.get_format_name(stream.get('stream_type')), | |
200 | 'ext': 'mp4', | |
201 | 'protocol': 'm3u8_native', | |
202 | 'filesize': int(stream.get('size')), | |
203 | 'width': stream.get('width'), | |
204 | 'height': stream.get('height'), | |
205 | } for stream in data['stream'] if stream.get('channel_type') != 'tail'] | |
206 | self._sort_formats(formats) | |
f9355dc9 | 207 | |
f1e66cb2 | 208 | return { |
f1e66cb2 YCH |
209 | 'id': video_id, |
210 | 'title': title, | |
59ed87cb | 211 | 'formats': formats, |
d16c0121 YCH |
212 | 'duration': video_data.get('seconds'), |
213 | 'thumbnail': video_data.get('logo'), | |
214 | 'uploader': video_data.get('username'), | |
215 | 'uploader_id': str_or_none(video_data.get('userid')), | |
216 | 'uploader_url': data.get('uploader', {}).get('homepage'), | |
217 | 'tags': video_data.get('tags'), | |
f1e66cb2 | 218 | } |
411c590a YCH |
219 | |
220 | ||
221 | class YoukuShowIE(InfoExtractor): | |
d3d4ba7f | 222 | _VALID_URL = r'https?://list\.youku\.com/show/id_(?P<id>[0-9a-z]+)\.html' |
411c590a YCH |
223 | IE_NAME = 'youku:show' |
224 | ||
225 | _TEST = { | |
d3d4ba7f | 226 | 'url': 'http://list.youku.com/show/id_zc7c670be07ff11e48b3f.html', |
411c590a YCH |
227 | 'info_dict': { |
228 | 'id': 'zc7c670be07ff11e48b3f', | |
229 | 'title': '花千骨 未删减版', | |
d3d4ba7f | 230 | 'description': 'md5:a1ae6f5618571bbeb5c9821f9c81b558', |
411c590a YCH |
231 | }, |
232 | 'playlist_count': 50, | |
233 | } | |
234 | ||
235 | _PAGE_SIZE = 40 | |
236 | ||
411c590a YCH |
237 | def _real_extract(self, url): |
238 | show_id = self._match_id(url) | |
239 | webpage = self._download_webpage(url, show_id) | |
240 | ||
d3d4ba7f YCH |
241 | entries = [] |
242 | page_config = self._parse_json(self._search_regex( | |
243 | r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'), | |
244 | show_id, transform_source=js_to_json) | |
245 | for idx in itertools.count(0): | |
246 | if idx == 0: | |
247 | playlist_data_url = 'http://list.youku.com/show/module' | |
248 | query = {'id': page_config['showid'], 'tab': 'point'} | |
249 | else: | |
250 | playlist_data_url = 'http://list.youku.com/show/point' | |
251 | query = { | |
252 | 'id': page_config['showid'], | |
253 | 'stage': 'reload_%d' % (self._PAGE_SIZE * idx + 1), | |
254 | } | |
255 | query['callback'] = 'cb' | |
256 | playlist_data = self._download_json( | |
257 | playlist_data_url, show_id, query=query, | |
258 | note='Downloading playlist data page %d' % (idx + 1), | |
259 | transform_source=lambda s: js_to_json(strip_jsonp(s)))['html'] | |
260 | video_urls = re.findall( | |
261 | r'<div[^>]+class="p-thumb"[^<]+<a[^>]+href="([^"]+)"', | |
262 | playlist_data) | |
263 | new_entries = [ | |
264 | self.url_result(urljoin(url, video_url), YoukuIE.ie_key()) | |
265 | for video_url in video_urls] | |
411c590a YCH |
266 | entries.extend(new_entries) |
267 | if len(new_entries) < self._PAGE_SIZE: | |
268 | break | |
269 | ||
d3d4ba7f YCH |
270 | desc = self._html_search_meta('description', webpage, fatal=False) |
271 | playlist_title = desc.split(',')[0] if desc else None | |
272 | detail_li = get_element_by_class('p-intro', webpage) | |
273 | playlist_description = get_element_by_class( | |
274 | 'intro-more', detail_li) if detail_li else None | |
275 | ||
276 | return self.playlist_result( | |
277 | entries, show_id, playlist_title, playlist_description) |