]>
Commit | Line | Data |
---|---|---|
5270d8cb | 1 | # encoding: utf-8 |
2 | ||
7dbf5ae5 | 3 | import os.path |
5270d8cb | 4 | import re |
5 | import json | |
6 | import hashlib | |
55f6597c | 7 | import uuid |
5270d8cb | 8 | |
9 | from .common import InfoExtractor | |
10 | from ..utils import ( | |
55f6597c | 11 | compat_urllib_parse, |
12 | compat_urllib_request, | |
693b8b2d | 13 | ExtractorError, |
7dbf5ae5 | 14 | url_basename, |
5270d8cb | 15 | ) |
16 | ||
17 | ||
18 | class SmotriIE(InfoExtractor): | |
19 | IE_DESC = u'Smotri.com' | |
20 | IE_NAME = u'smotri' | |
aaebed13 PH |
21 | _VALID_URL = r'^https?://(?:www\.)?(?P<url>smotri\.com/video/view/\?id=(?P<videoid>v(?P<realvideoid>[0-9]+)[a-z0-9]{4}))' |
22 | ||
5270d8cb | 23 | _TESTS = [ |
24 | # real video id 2610366 | |
25 | { | |
26 | u'url': u'http://smotri.com/video/view/?id=v261036632ab', | |
27 | u'file': u'v261036632ab.mp4', | |
aaebed13 | 28 | u'md5': u'2a7b08249e6f5636557579c368040eb9', |
5270d8cb | 29 | u'info_dict': { |
30 | u'title': u'катастрофа с камер видеонаблюдения', | |
31 | u'uploader': u'rbc2008', | |
32 | u'uploader_id': u'rbc08', | |
33 | u'upload_date': u'20131118', | |
aaebed13 PH |
34 | u'description': u'катастрофа с камер видеонаблюдения, видео катастрофа с камер видеонаблюдения', |
35 | u'thumbnail': u'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg', | |
5270d8cb | 36 | }, |
37 | }, | |
38 | # real video id 57591 | |
39 | { | |
40 | u'url': u'http://smotri.com/video/view/?id=v57591cb20', | |
41 | u'file': u'v57591cb20.flv', | |
aaebed13 | 42 | u'md5': u'830266dfc21f077eac5afd1883091bcd', |
5270d8cb | 43 | u'info_dict': { |
44 | u'title': u'test', | |
45 | u'uploader': u'Support Photofile@photofile', | |
46 | u'uploader_id': u'support-photofile', | |
47 | u'upload_date': u'20070704', | |
aaebed13 PH |
48 | u'description': u'test, видео test', |
49 | u'thumbnail': u'http://frame4.loadup.ru/03/ed/57591.2.3.jpg', | |
50 | }, | |
5270d8cb | 51 | }, |
52 | # video-password | |
53 | { | |
54 | u'url': u'http://smotri.com/video/view/?id=v1390466a13c', | |
55 | u'file': u'v1390466a13c.mp4', | |
aaebed13 | 56 | u'md5': u'f6331cef33cad65a0815ee482a54440b', |
5270d8cb | 57 | u'info_dict': { |
58 | u'title': u'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1', | |
59 | u'uploader': u'timoxa40', | |
60 | u'uploader_id': u'timoxa40', | |
61 | u'upload_date': u'20100404', | |
aaebed13 PH |
62 | u'thumbnail': u'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg', |
63 | u'description': u'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1, видео TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1', | |
5270d8cb | 64 | }, |
65 | u'params': { | |
66 | u'videopassword': u'qwerty', | |
67 | }, | |
68 | }, | |
69 | # age limit + video-password | |
70 | { | |
71 | u'url': u'http://smotri.com/video/view/?id=v15408898bcf', | |
72 | u'file': u'v15408898bcf.flv', | |
aaebed13 | 73 | u'md5': u'91e909c9f0521adf5ee86fbe073aad70', |
5270d8cb | 74 | u'info_dict': { |
75 | u'title': u'этот ролик не покажут по ТВ', | |
76 | u'uploader': u'zzxxx', | |
77 | u'uploader_id': u'ueggb', | |
78 | u'upload_date': u'20101001', | |
79 | u'thumbnail': u'http://frame3.loadup.ru/75/75/1540889.1.3.jpg', | |
aaebed13 PH |
80 | u'age_limit': 18, |
81 | u'description': u'этот ролик не покажут по ТВ, видео этот ролик не покажут по ТВ', | |
82 | }, | |
5270d8cb | 83 | u'params': { |
84 | u'videopassword': u'333' | |
85 | } | |
86 | } | |
87 | ] | |
88 | ||
89 | _SUCCESS = 0 | |
90 | _PASSWORD_NOT_VERIFIED = 1 | |
91 | _PASSWORD_DETECTED = 2 | |
92 | _VIDEO_NOT_FOUND = 3 | |
aaebed13 | 93 | |
5270d8cb | 94 | def _search_meta(self, name, html, display_name=None): |
95 | if display_name is None: | |
96 | display_name = name | |
97 | return self._html_search_regex( | |
98 | r'<meta itemprop="%s" content="([^"]+)" />' % re.escape(name), | |
99 | html, display_name, fatal=False) | |
aaebed13 PH |
100 | return self._html_search_meta(name, html, display_name) |
101 | ||
5270d8cb | 102 | def _real_extract(self, url): |
103 | mobj = re.match(self._VALID_URL, url) | |
104 | video_id = mobj.group('videoid') | |
105 | real_video_id = mobj.group('realvideoid') | |
106 | ||
107 | # Download video JSON data | |
108 | video_json_url = 'http://smotri.com/vt.php?id=%s' % real_video_id | |
109 | video_json_page = self._download_webpage(video_json_url, video_id, u'Downloading video JSON') | |
110 | video_json = json.loads(video_json_page) | |
111 | ||
112 | status = video_json['status'] | |
113 | if status == self._VIDEO_NOT_FOUND: | |
114 | raise ExtractorError(u'Video %s does not exist' % video_id, expected=True) | |
aaebed13 | 115 | elif status == self._PASSWORD_DETECTED: # The video is protected by a password, retry with |
5270d8cb | 116 | # video-password set |
117 | video_password = self._downloader.params.get('videopassword', None) | |
118 | if not video_password: | |
119 | raise ExtractorError(u'This video is protected by a password, use the --video-password option', expected=True) | |
aaebed13 | 120 | video_json_url += '&md5pass=%s' % hashlib.md5(video_password.encode('utf-8')).hexdigest() |
5270d8cb | 121 | video_json_page = self._download_webpage(video_json_url, video_id, u'Downloading video JSON (video-password set)') |
122 | video_json = json.loads(video_json_page) | |
123 | status = video_json['status'] | |
124 | if status == self._PASSWORD_NOT_VERIFIED: | |
125 | raise ExtractorError(u'Video password is invalid', expected=True) | |
126 | ||
127 | if status != self._SUCCESS: | |
128 | raise ExtractorError(u'Unexpected status value %s' % status) | |
129 | ||
130 | # Extract the URL of the video | |
131 | video_url = video_json['file_data'] | |
5270d8cb | 132 | |
133 | # Video JSON does not provide enough meta data | |
134 | # We will extract some from the video web page instead | |
135 | video_page_url = 'http://' + mobj.group('url') | |
136 | video_page = self._download_webpage(video_page_url, video_id, u'Downloading video page') | |
7dbf5ae5 PH |
137 | |
138 | # Warning if video is unavailable | |
139 | warning = self._html_search_regex( | |
140 | r'<div class="videoUnModer">(.*?)</div>', video_page, | |
f7e9d77f | 141 | u'warning message', default=None) |
7dbf5ae5 PH |
142 | if warning is not None: |
143 | self._downloader.report_warning( | |
144 | u'Video %s may not be available; smotri said: %s ' % | |
145 | (video_id, warning)) | |
146 | ||
5270d8cb | 147 | # Adult content |
148 | if re.search(u'EroConfirmText">', video_page) is not None: | |
149 | self.report_age_confirmation() | |
150 | confirm_string = self._html_search_regex( | |
aaebed13 | 151 | r'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id, |
5270d8cb | 152 | video_page, u'confirm string') |
153 | confirm_url = video_page_url + '&confirm=%s' % confirm_string | |
154 | video_page = self._download_webpage(confirm_url, video_id, u'Downloading video page (age confirmed)') | |
155 | adult_content = True | |
156 | else: | |
157 | adult_content = False | |
158 | ||
159 | # Extract the rest of meta data | |
160 | video_title = self._search_meta(u'name', video_page, u'title') | |
161 | if not video_title: | |
7dbf5ae5 | 162 | video_title = os.path.splitext(url_basename(video_url))[0] |
aaebed13 | 163 | |
5270d8cb | 164 | video_description = self._search_meta(u'description', video_page) |
aaebed13 | 165 | END_TEXT = u' на сайте Smotri.com' |
8d0bdeba | 166 | if video_description and video_description.endswith(END_TEXT): |
aaebed13 PH |
167 | video_description = video_description[:-len(END_TEXT)] |
168 | START_TEXT = u'Смотреть онлайн ролик ' | |
8d0bdeba | 169 | if video_description and video_description.startswith(START_TEXT): |
aaebed13 | 170 | video_description = video_description[len(START_TEXT):] |
5270d8cb | 171 | video_thumbnail = self._search_meta(u'thumbnail', video_page) |
aaebed13 PH |
172 | |
173 | upload_date_str = self._search_meta(u'uploadDate', video_page, u'upload date') | |
8d0bdeba PH |
174 | if upload_date_str: |
175 | upload_date_m = re.search(r'(?P<year>\d{4})\.(?P<month>\d{2})\.(?P<day>\d{2})T', upload_date_str) | |
176 | video_upload_date = ( | |
177 | ( | |
178 | upload_date_m.group('year') + | |
179 | upload_date_m.group('month') + | |
180 | upload_date_m.group('day') | |
181 | ) | |
182 | if upload_date_m else None | |
5270d8cb | 183 | ) |
8d0bdeba PH |
184 | else: |
185 | video_upload_date = None | |
5270d8cb | 186 | |
187 | duration_str = self._search_meta(u'duration', video_page) | |
8d0bdeba PH |
188 | if duration_str: |
189 | duration_m = re.search(r'T(?P<hours>[0-9]{2})H(?P<minutes>[0-9]{2})M(?P<seconds>[0-9]{2})S', duration_str) | |
190 | video_duration = ( | |
191 | ( | |
192 | (int(duration_m.group('hours')) * 60 * 60) + | |
193 | (int(duration_m.group('minutes')) * 60) + | |
194 | int(duration_m.group('seconds')) | |
195 | ) | |
196 | if duration_m else None | |
5270d8cb | 197 | ) |
8d0bdeba PH |
198 | else: |
199 | video_duration = None | |
5270d8cb | 200 | |
201 | video_uploader = self._html_search_regex( | |
aaebed13 | 202 | u'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info[^"]+">(.*?)</a>', |
5270d8cb | 203 | video_page, u'uploader', fatal=False, flags=re.MULTILINE|re.DOTALL) |
204 | ||
205 | video_uploader_id = self._html_search_regex( | |
aaebed13 | 206 | u'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info\\(.*?\'([^\']+)\'\\);">', |
5270d8cb | 207 | video_page, u'uploader id', fatal=False, flags=re.MULTILINE|re.DOTALL) |
208 | ||
209 | video_view_count = self._html_search_regex( | |
aaebed13 | 210 | u'Общее количество просмотров.*?<span class="Number">(\\d+)</span>', |
5270d8cb | 211 | video_page, u'view count', fatal=False, flags=re.MULTILINE|re.DOTALL) |
212 | ||
213 | return { | |
214 | 'id': video_id, | |
215 | 'url': video_url, | |
216 | 'title': video_title, | |
5270d8cb | 217 | 'thumbnail': video_thumbnail, |
218 | 'description': video_description, | |
219 | 'uploader': video_uploader, | |
220 | 'upload_date': video_upload_date, | |
221 | 'uploader_id': video_uploader_id, | |
0cc83dc5 | 222 | 'duration': video_duration, |
5270d8cb | 223 | 'view_count': video_view_count, |
224 | 'age_limit': 18 if adult_content else 0, | |
225 | 'video_page_url': video_page_url | |
226 | } | |
227 | ||
aaebed13 | 228 | |
5270d8cb | 229 | class SmotriCommunityIE(InfoExtractor): |
230 | IE_DESC = u'Smotri.com community videos' | |
231 | IE_NAME = u'smotri:community' | |
aaebed13 | 232 | _VALID_URL = r'^https?://(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)' |
5270d8cb | 233 | |
234 | def _real_extract(self, url): | |
235 | mobj = re.match(self._VALID_URL, url) | |
236 | community_id = mobj.group('communityid') | |
aaebed13 | 237 | |
5270d8cb | 238 | url = 'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id |
239 | rss = self._download_xml(url, community_id, u'Downloading community RSS') | |
aaebed13 | 240 | |
5270d8cb | 241 | entries = [self.url_result(video_url.text, 'Smotri') |
242 | for video_url in rss.findall('./channel/item/link')] | |
aaebed13 PH |
243 | |
244 | description_text = rss.find('./channel/description').text | |
5270d8cb | 245 | community_title = self._html_search_regex( |
aaebed13 | 246 | u'^Видео сообщества "([^"]+)"$', description_text, u'community title') |
5270d8cb | 247 | |
248 | return self.playlist_result(entries, community_id, community_title) | |
aaebed13 PH |
249 | |
250 | ||
5270d8cb | 251 | class SmotriUserIE(InfoExtractor): |
252 | IE_DESC = u'Smotri.com user videos' | |
253 | IE_NAME = u'smotri:user' | |
aaebed13 PH |
254 | _VALID_URL = r'^https?://(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)' |
255 | ||
5270d8cb | 256 | def _real_extract(self, url): |
aaebed13 | 257 | mobj = re.match(self._VALID_URL, url) |
5270d8cb | 258 | user_id = mobj.group('userid') |
aaebed13 | 259 | |
5270d8cb | 260 | url = 'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id |
261 | rss = self._download_xml(url, user_id, u'Downloading user RSS') | |
aaebed13 | 262 | |
5270d8cb | 263 | entries = [self.url_result(video_url.text, 'Smotri') |
264 | for video_url in rss.findall('./channel/item/link')] | |
aaebed13 PH |
265 | |
266 | description_text = rss.find('./channel/description').text | |
5270d8cb | 267 | user_nickname = self._html_search_regex( |
aaebed13 PH |
268 | u'^Видео режиссера (.*)$', description_text, |
269 | u'user nickname') | |
5270d8cb | 270 | |
271 | return self.playlist_result(entries, user_id, user_nickname) | |
55f6597c | 272 | |
273 | ||
274 | class SmotriBroadcastIE(InfoExtractor): | |
275 | IE_DESC = u'Smotri.com broadcasts' | |
276 | IE_NAME = u'smotri:broadcast' | |
277 | _VALID_URL = r'^https?://(?:www\.)?(?P<url>smotri\.com/live/(?P<broadcastid>[^/]+))/?.*' | |
278 | ||
279 | def _real_extract(self, url): | |
280 | mobj = re.match(self._VALID_URL, url) | |
281 | broadcast_id = mobj.group('broadcastid') | |
282 | ||
283 | broadcast_url = 'http://' + mobj.group('url') | |
284 | broadcast_page = self._download_webpage(broadcast_url, broadcast_id, u'Downloading broadcast page') | |
285 | ||
286 | if re.search(u'>Режиссер с логином <br/>"%s"<br/> <span>не существует<' % broadcast_id, broadcast_page) is not None: | |
287 | raise ExtractorError(u'Broadcast %s does not exist' % broadcast_id, expected=True) | |
288 | ||
289 | # Adult content | |
290 | if re.search(u'EroConfirmText">', broadcast_page) is not None: | |
291 | ||
292 | (username, password) = self._get_login_info() | |
293 | if username is None: | |
294 | raise ExtractorError(u'Erotic broadcasts allowed only for registered users, ' | |
295 | u'use --username and --password options to provide account credentials.', expected=True) | |
296 | ||
297 | # Log in | |
298 | login_form_strs = { | |
299 | u'login-hint53': '1', | |
300 | u'confirm_erotic': '1', | |
301 | u'login': username, | |
302 | u'password': password, | |
303 | } | |
304 | # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode | |
305 | # chokes on unicode | |
306 | login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items()) | |
307 | login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8') | |
308 | login_url = broadcast_url + '/?no_redirect=1' | |
309 | request = compat_urllib_request.Request(login_url, login_data) | |
310 | request.add_header('Content-Type', 'application/x-www-form-urlencoded') | |
311 | broadcast_page = self._download_webpage( | |
312 | request, broadcast_id, note=u'Logging in and confirming age') | |
313 | ||
314 | if re.search(u'>Неверный логин или пароль<', broadcast_page) is not None: | |
315 | raise ExtractorError(u'Unable to log in: bad username or password', expected=True) | |
316 | ||
317 | adult_content = True | |
318 | else: | |
319 | adult_content = False | |
320 | ||
321 | ticket = self._html_search_regex( | |
8aff7b9b | 322 | u'window\.broadcast_control\.addFlashVar\\(\'file\', \'([^\']+)\'\\);', |
55f6597c | 323 | broadcast_page, u'broadcast ticket') |
324 | ||
325 | url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket | |
326 | ||
327 | broadcast_password = self._downloader.params.get('videopassword', None) | |
328 | if broadcast_password: | |
329 | url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest() | |
330 | ||
331 | broadcast_json_page = self._download_webpage(url, broadcast_id, u'Downloading broadcast JSON') | |
332 | ||
333 | try: | |
334 | broadcast_json = json.loads(broadcast_json_page) | |
335 | ||
336 | protected_broadcast = broadcast_json['_pass_protected'] == 1 | |
337 | if protected_broadcast and not broadcast_password: | |
338 | raise ExtractorError(u'This broadcast is protected by a password, use the --video-password option', expected=True) | |
339 | ||
340 | broadcast_offline = broadcast_json['is_play'] == 0 | |
341 | if broadcast_offline: | |
342 | raise ExtractorError(u'Broadcast %s is offline' % broadcast_id, expected=True) | |
343 | ||
344 | rtmp_url = broadcast_json['_server'] | |
345 | if not rtmp_url.startswith('rtmp://'): | |
346 | raise ExtractorError(u'Unexpected broadcast rtmp URL') | |
347 | ||
348 | broadcast_playpath = broadcast_json['_streamName'] | |
349 | broadcast_thumbnail = broadcast_json['_imgURL'] | |
350 | broadcast_title = broadcast_json['title'] | |
351 | broadcast_description = broadcast_json['description'] | |
352 | broadcaster_nick = broadcast_json['nick'] | |
353 | broadcaster_login = broadcast_json['login'] | |
354 | rtmp_conn = 'S:%s' % uuid.uuid4().hex | |
355 | except KeyError: | |
356 | if protected_broadcast: | |
357 | raise ExtractorError(u'Bad broadcast password', expected=True) | |
358 | raise ExtractorError(u'Unexpected broadcast JSON') | |
359 | ||
360 | return { | |
361 | 'id': broadcast_id, | |
362 | 'url': rtmp_url, | |
363 | 'title': broadcast_title, | |
364 | 'thumbnail': broadcast_thumbnail, | |
365 | 'description': broadcast_description, | |
366 | 'uploader': broadcaster_nick, | |
367 | 'uploader_id': broadcaster_login, | |
368 | 'age_limit': 18 if adult_content else 0, | |
369 | 'ext': 'flv', | |
370 | 'play_path': broadcast_playpath, | |
371 | 'rtmp_live': True, | |
372 | 'rtmp_conn': rtmp_conn | |
693b8b2d | 373 | } |