]> jfr.im git - yt-dlp.git/blob - youtube_dl/InfoExtractors.py
projects / yt-dlp.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Adds format listing/selection support to the Comedy Central extractor.
[yt-dlp.git] / youtube_dl / InfoExtractors.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import datetime
5 import HTMLParser
6 import httplib
7 import netrc
8 import os
9 import re
10 import socket
11 import time
12 import urllib
13 import urllib2
14 import email.utils
15 import xml.etree.ElementTree
16 import random
17 import math
18 from urlparse import parse_qs
19
20 try:
21 import cStringIO as StringIO
22 except ImportError:
23 import StringIO
24
25 from utils import *
26
27
28 class InfoExtractor(object):
29 """Information Extractor class.
30
31 Information extractors are the classes that, given a URL, extract
32 information from the video (or videos) the URL refers to. This
33 information includes the real video URL, the video title and simplified
34 title, author and others. The information is stored in a dictionary
35 which is then passed to the FileDownloader. The FileDownloader
36 processes this information possibly downloading the video to the file
37 system, among other possible outcomes. The dictionaries must include
38 the following fields:
39
40 id: Video identifier.
41 url: Final video URL.
42 uploader: Nickname of the video uploader.
43 title: Literal title.
44 ext: Video filename extension.
45 format: Video format.
46 player_url: SWF Player URL (may be None).
47
48 The following fields are optional. Their primary purpose is to allow
49 youtube-dl to serve as the backend for a video search function, such
50 as the one in youtube2mp3. They are only used when their respective
51 forced printing functions are called:
52
53 thumbnail: Full URL to a video thumbnail image.
54 description: One-line video description.
55
56 Subclasses of this one should re-define the _real_initialize() and
57 _real_extract() methods and define a _VALID_URL regexp.
58 Probably, they should also be added to the list of extractors.
59 """
60
61 _ready = False
62 _downloader = None
63
64 def __init__(self, downloader=None):
65 """Constructor. Receives an optional downloader."""
66 self._ready = False
67 self.set_downloader(downloader)
68
69 def suitable(self, url):
70 """Receives a URL and returns True if suitable for this IE."""
71 return re.match(self._VALID_URL, url) is not None
72
73 def initialize(self):
74 """Initializes an instance (authentication, etc)."""
75 if not self._ready:
76 self._real_initialize()
77 self._ready = True
78
79 def extract(self, url):
80 """Extracts URL information and returns it in list of dicts."""
81 self.initialize()
82 return self._real_extract(url)
83
84 def set_downloader(self, downloader):
85 """Sets the downloader for this IE."""
86 self._downloader = downloader
87
88 def _real_initialize(self):
89 """Real initialization process. Redefine in subclasses."""
90 pass
91
92 def _real_extract(self, url):
93 """Real extraction process. Redefine in subclasses."""
94 pass
95
96
97 class YoutubeIE(InfoExtractor):
98 """Information extractor for youtube.com."""
99
100 _VALID_URL = r"""^
101 (
102 (?:https?://)? # http(s):// (optional)
103 (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
104 tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains
105 (?:.*?\#/)? # handle anchor (#/) redirect urls
106 (?!view_play_list|my_playlists|artist|playlist) # ignore playlist URLs
107 (?: # the various things that can precede the ID:
108 (?:(?:v|embed|e)/) # v/ or embed/ or e/
109 |(?: # or the v= param in all its forms
110 (?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
111 (?:\?|\#!?) # the params delimiter ? or # or #!
112 (?:.+&)? # any other preceding param (like /?s=tuff&v=xxxx)
113 v=
114 )
115 )? # optional -> youtube.com/xxxx is OK
116 )? # all until now is optional -> you can pass the naked ID
117 ([0-9A-Za-z_-]+) # here is it! the YouTube video ID
118 (?(1).+)? # if we found the ID, everything can follow
119 $"""
120 _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
121 _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
122 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
123 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
124 _NETRC_MACHINE = 'youtube'
125 # Listed in order of quality
126 _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
127 _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
128 _video_extensions = {
129 '13': '3gp',
130 '17': 'mp4',
131 '18': 'mp4',
132 '22': 'mp4',
133 '37': 'mp4',
134 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
135 '43': 'webm',
136 '44': 'webm',
137 '45': 'webm',
138 '46': 'webm',
139 }
140 _video_dimensions = {
141 '5': '240x400',
142 '6': '???',
143 '13': '???',
144 '17': '144x176',
145 '18': '360x640',
146 '22': '720x1280',
147 '34': '360x640',
148 '35': '480x854',
149 '37': '1080x1920',
150 '38': '3072x4096',
151 '43': '360x640',
152 '44': '480x854',
153 '45': '720x1280',
154 '46': '1080x1920',
155 }
156 IE_NAME = u'youtube'
157
158 def suitable(self, url):
159 """Receives a URL and returns True if suitable for this IE."""
160 return re.match(self._VALID_URL, url, re.VERBOSE) is not None
161
162 def report_lang(self):
163 """Report attempt to set language."""
164 self._downloader.to_screen(u'[youtube] Setting language')
165
166 def report_login(self):
167 """Report attempt to log in."""
168 self._downloader.to_screen(u'[youtube] Logging in')
169
170 def report_age_confirmation(self):
171 """Report attempt to confirm age."""
172 self._downloader.to_screen(u'[youtube] Confirming age')
173
174 def report_video_webpage_download(self, video_id):
175 """Report attempt to download video webpage."""
176 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
177
178 def report_video_info_webpage_download(self, video_id):
179 """Report attempt to download video info webpage."""
180 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
181
182 def report_video_subtitles_download(self, video_id):
183 """Report attempt to download video info webpage."""
184 self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id)
185
186 def report_information_extraction(self, video_id):
187 """Report attempt to extract video information."""
188 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
189
190 def report_unavailable_format(self, video_id, format):
191 """Report extracted video URL."""
192 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
193
194 def report_rtmp_download(self):
195 """Indicate the download will use the RTMP protocol."""
196 self._downloader.to_screen(u'[youtube] RTMP download detected')
197
198 def _closed_captions_xml_to_srt(self, xml_string):
199 srt = ''
200 texts = re.findall(r'<text start="([\d\.]+)"( dur="([\d\.]+)")?>([^<]+)</text>', xml_string, re.MULTILINE)
201 # TODO parse xml instead of regex
202 for n, (start, dur_tag, dur, caption) in enumerate(texts):
203 if not dur: dur = '4'
204 start = float(start)
205 end = start + float(dur)
206 start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
207 end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
208 caption = unescapeHTML(caption)
209 caption = unescapeHTML(caption) # double cycle, intentional
210 srt += str(n+1) + '\n'
211 srt += start + ' --> ' + end + '\n'
212 srt += caption + '\n\n'
213 return srt
214
215 def _print_formats(self, formats):
216 print('Available formats:')
217 for x in formats:
218 print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
219
220 def _real_initialize(self):
221 if self._downloader is None:
222 return
223
224 username = None
225 password = None
226 downloader_params = self._downloader.params
227
228 # Attempt to use provided username and password or .netrc data
229 if downloader_params.get('username', None) is not None:
230 username = downloader_params['username']
231 password = downloader_params['password']
232 elif downloader_params.get('usenetrc', False):
233 try:
234 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
235 if info is not None:
236 username = info[0]
237 password = info[2]
238 else:
239 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
240 except (IOError, netrc.NetrcParseError), err:
241 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
242 return
243
244 # Set language
245 request = urllib2.Request(self._LANG_URL)
246 try:
247 self.report_lang()
248 urllib2.urlopen(request).read()
249 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
250 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
251 return
252
253 # No authentication to be performed
254 if username is None:
255 return
256
257 # Log in
258 login_form = {
259 'current_form': 'loginForm',
260 'next': '/',
261 'action_login': 'Log In',
262 'username': username,
263 'password': password,
264 }
265 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
266 try:
267 self.report_login()
268 login_results = urllib2.urlopen(request).read()
269 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
270 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
271 return
272 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
273 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
274 return
275
276 # Confirm age
277 age_form = {
278 'next_url': '/',
279 'action_confirm': 'Confirm',
280 }
281 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
282 try:
283 self.report_age_confirmation()
284 age_results = urllib2.urlopen(request).read()
285 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
286 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
287 return
288
289 def _real_extract(self, url):
290 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
291 mobj = re.search(self._NEXT_URL_RE, url)
292 if mobj:
293 url = 'http://www.youtube.com/' + urllib.unquote(mobj.group(1)).lstrip('/')
294
295 # Extract video id from URL
296 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
297 if mobj is None:
298 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
299 return
300 video_id = mobj.group(2)
301
302 # Get video webpage
303 self.report_video_webpage_download(video_id)
304 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
305 try:
306 video_webpage = urllib2.urlopen(request).read()
307 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
308 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
309 return
310
311 # Attempt to extract SWF player URL
312 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
313 if mobj is not None:
314 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
315 else:
316 player_url = None
317
318 # Get video info
319 self.report_video_info_webpage_download(video_id)
320 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
321 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
322 % (video_id, el_type))
323 request = urllib2.Request(video_info_url)
324 try:
325 video_info_webpage = urllib2.urlopen(request).read()
326 video_info = parse_qs(video_info_webpage)
327 if 'token' in video_info:
328 break
329 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
330 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
331 return
332 if 'token' not in video_info:
333 if 'reason' in video_info:
334 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
335 else:
336 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
337 return
338
339 # Check for "rental" videos
340 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
341 self._downloader.trouble(u'ERROR: "rental" videos not supported')
342 return
343
344 # Start extracting information
345 self.report_information_extraction(video_id)
346
347 # uploader
348 if 'author' not in video_info:
349 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
350 return
351 video_uploader = urllib.unquote_plus(video_info['author'][0])
352
353 # title
354 if 'title' not in video_info:
355 self._downloader.trouble(u'ERROR: unable to extract video title')
356 return
357 video_title = urllib.unquote_plus(video_info['title'][0])
358 video_title = video_title.decode('utf-8')
359
360 # thumbnail image
361 if 'thumbnail_url' not in video_info:
362 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
363 video_thumbnail = ''
364 else: # don't panic if we can't find it
365 video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
366
367 # upload date
368 upload_date = u'NA'
369 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
370 if mobj is not None:
371 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
372 format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
373 for expression in format_expressions:
374 try:
375 upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
376 except:
377 pass
378
379 # description
380 video_description = get_element_by_id("eow-description", video_webpage.decode('utf8'))
381 if video_description: video_description = clean_html(video_description)
382 else: video_description = ''
383
384 # closed captions
385 video_subtitles = None
386 if self._downloader.params.get('writesubtitles', False):
387 try:
388 self.report_video_subtitles_download(video_id)
389 request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
390 try:
391 srt_list = urllib2.urlopen(request).read()
392 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
393 raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
394 srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
395 srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
396 if not srt_lang_list:
397 raise Trouble(u'WARNING: video has no closed captions')
398 if self._downloader.params.get('subtitleslang', False):
399 srt_lang = self._downloader.params.get('subtitleslang')
400 elif 'en' in srt_lang_list:
401 srt_lang = 'en'
402 else:
403 srt_lang = srt_lang_list.keys()[0]
404 if not srt_lang in srt_lang_list:
405 raise Trouble(u'WARNING: no closed captions found in the specified language')
406 request = urllib2.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
407 try:
408 srt_xml = urllib2.urlopen(request).read()
409 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
410 raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
411 if not srt_xml:
412 raise Trouble(u'WARNING: unable to download video subtitles')
413 video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
414 except Trouble as trouble:
415 self._downloader.trouble(trouble[0])
416
417 if 'length_seconds' not in video_info:
418 self._downloader.trouble(u'WARNING: unable to extract video duration')
419 video_duration = ''
420 else:
421 video_duration = urllib.unquote_plus(video_info['length_seconds'][0])
422
423 # token
424 video_token = urllib.unquote_plus(video_info['token'][0])
425
426 # Decide which formats to download
427 req_format = self._downloader.params.get('format', None)
428
429 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
430 self.report_rtmp_download()
431 video_url_list = [(None, video_info['conn'][0])]
432 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
433 url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
434 url_data = [parse_qs(uds) for uds in url_data_strs]
435 url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
436 url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
437
438 format_limit = self._downloader.params.get('format_limit', None)
439 available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
440 if format_limit is not None and format_limit in available_formats:
441 format_list = available_formats[available_formats.index(format_limit):]
442 else:
443 format_list = available_formats
444 existing_formats = [x for x in format_list if x in url_map]
445 if len(existing_formats) == 0:
446 self._downloader.trouble(u'ERROR: no known formats available for video')
447 return
448 if self._downloader.params.get('listformats', None):
449 self._print_formats(existing_formats)
450 return
451 if req_format is None or req_format == 'best':
452 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
453 elif req_format == 'worst':
454 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
455 elif req_format in ('-1', 'all'):
456 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
457 else:
458 # Specific formats. We pick the first in a slash-delimeted sequence.
459 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
460 req_formats = req_format.split('/')
461 video_url_list = None
462 for rf in req_formats:
463 if rf in url_map:
464 video_url_list = [(rf, url_map[rf])]
465 break
466 if video_url_list is None:
467 self._downloader.trouble(u'ERROR: requested format not available')
468 return
469 else:
470 self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
471 return
472
473 results = []
474 for format_param, video_real_url in video_url_list:
475 # Extension
476 video_extension = self._video_extensions.get(format_param, 'flv')
477
478 results.append({
479 'id': video_id.decode('utf-8'),
480 'url': video_real_url.decode('utf-8'),
481 'uploader': video_uploader.decode('utf-8'),
482 'upload_date': upload_date,
483 'title': video_title,
484 'ext': video_extension.decode('utf-8'),
485 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
486 'thumbnail': video_thumbnail.decode('utf-8'),
487 'description': video_description,
488 'player_url': player_url,
489 'subtitles': video_subtitles,
490 'duration': video_duration
491 })
492 return results
493
494
495 class MetacafeIE(InfoExtractor):
496 """Information Extractor for metacafe.com."""
497
498 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
499 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
500 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
501 IE_NAME = u'metacafe'
502
503 def __init__(self, downloader=None):
504 InfoExtractor.__init__(self, downloader)
505
506 def report_disclaimer(self):
507 """Report disclaimer retrieval."""
508 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
509
510 def report_age_confirmation(self):
511 """Report attempt to confirm age."""
512 self._downloader.to_screen(u'[metacafe] Confirming age')
513
514 def report_download_webpage(self, video_id):
515 """Report webpage download."""
516 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
517
518 def report_extraction(self, video_id):
519 """Report information extraction."""
520 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
521
522 def _real_initialize(self):
523 # Retrieve disclaimer
524 request = urllib2.Request(self._DISCLAIMER)
525 try:
526 self.report_disclaimer()
527 disclaimer = urllib2.urlopen(request).read()
528 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
529 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
530 return
531
532 # Confirm age
533 disclaimer_form = {
534 'filters': '0',
535 'submit': "Continue - I'm over 18",
536 }
537 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
538 try:
539 self.report_age_confirmation()
540 disclaimer = urllib2.urlopen(request).read()
541 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
542 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
543 return
544
545 def _real_extract(self, url):
546 # Extract id and simplified title from URL
547 mobj = re.match(self._VALID_URL, url)
548 if mobj is None:
549 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
550 return
551
552 video_id = mobj.group(1)
553
554 # Check if video comes from YouTube
555 mobj2 = re.match(r'^yt-(.*)$', video_id)
556 if mobj2 is not None:
557 self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
558 return
559
560 # Retrieve video webpage to extract further information
561 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
562 try:
563 self.report_download_webpage(video_id)
564 webpage = urllib2.urlopen(request).read()
565 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
566 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
567 return
568
569 # Extract URL, uploader and title from webpage
570 self.report_extraction(video_id)
571 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
572 if mobj is not None:
573 mediaURL = urllib.unquote(mobj.group(1))
574 video_extension = mediaURL[-3:]
575
576 # Extract gdaKey if available
577 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
578 if mobj is None:
579 video_url = mediaURL
580 else:
581 gdaKey = mobj.group(1)
582 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
583 else:
584 mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
585 if mobj is None:
586 self._downloader.trouble(u'ERROR: unable to extract media URL')
587 return
588 vardict = parse_qs(mobj.group(1))
589 if 'mediaData' not in vardict:
590 self._downloader.trouble(u'ERROR: unable to extract media URL')
591 return
592 mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
593 if mobj is None:
594 self._downloader.trouble(u'ERROR: unable to extract media URL')
595 return
596 mediaURL = mobj.group(1).replace('\\/', '/')
597 video_extension = mediaURL[-3:]
598 video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
599
600 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
601 if mobj is None:
602 self._downloader.trouble(u'ERROR: unable to extract title')
603 return
604 video_title = mobj.group(1).decode('utf-8')
605
606 mobj = re.search(r'submitter=(.*?);', webpage)
607 if mobj is None:
608 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
609 return
610 video_uploader = mobj.group(1)
611
612 return [{
613 'id': video_id.decode('utf-8'),
614 'url': video_url.decode('utf-8'),
615 'uploader': video_uploader.decode('utf-8'),
616 'upload_date': u'NA',
617 'title': video_title,
618 'ext': video_extension.decode('utf-8'),
619 'format': u'NA',
620 'player_url': None,
621 }]
622
623
624 class DailymotionIE(InfoExtractor):
625 """Information Extractor for Dailymotion"""
626
627 _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
628 IE_NAME = u'dailymotion'
629
630 def __init__(self, downloader=None):
631 InfoExtractor.__init__(self, downloader)
632
633 def report_download_webpage(self, video_id):
634 """Report webpage download."""
635 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
636
637 def report_extraction(self, video_id):
638 """Report information extraction."""
639 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
640
641 def _real_extract(self, url):
642 # Extract id and simplified title from URL
643 mobj = re.match(self._VALID_URL, url)
644 if mobj is None:
645 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
646 return
647
648 video_id = mobj.group(1).split('_')[0].split('?')[0]
649
650 video_extension = 'mp4'
651
652 # Retrieve video webpage to extract further information
653 request = urllib2.Request(url)
654 request.add_header('Cookie', 'family_filter=off')
655 try:
656 self.report_download_webpage(video_id)
657 webpage = urllib2.urlopen(request).read()
658 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
659 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
660 return
661
662 # Extract URL, uploader and title from webpage
663 self.report_extraction(video_id)
664 mobj = re.search(r'\s*var flashvars = (.*)', webpage)
665 if mobj is None:
666 self._downloader.trouble(u'ERROR: unable to extract media URL')
667 return
668 flashvars = urllib.unquote(mobj.group(1))
669
670 for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
671 if key in flashvars:
672 max_quality = key
673 self._downloader.to_screen(u'[dailymotion] Using %s' % key)
674 break
675 else:
676 self._downloader.trouble(u'ERROR: unable to extract video URL')
677 return
678
679 mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
680 if mobj is None:
681 self._downloader.trouble(u'ERROR: unable to extract video URL')
682 return
683
684 video_url = urllib.unquote(mobj.group(1)).replace('\\/', '/')
685
686 # TODO: support choosing qualities
687
688 mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
689 if mobj is None:
690 self._downloader.trouble(u'ERROR: unable to extract title')
691 return
692 video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
693
694 video_uploader = u'NA'
695 mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
696 if mobj is None:
697 # lookin for official user
698 mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage)
699 if mobj_official is None:
700 self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
701 else:
702 video_uploader = mobj_official.group(1)
703 else:
704 video_uploader = mobj.group(1)
705
706 video_upload_date = u'NA'
707 mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
708 if mobj is not None:
709 video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
710
711 return [{
712 'id': video_id.decode('utf-8'),
713 'url': video_url.decode('utf-8'),
714 'uploader': video_uploader.decode('utf-8'),
715 'upload_date': video_upload_date,
716 'title': video_title,
717 'ext': video_extension.decode('utf-8'),
718 'format': u'NA',
719 'player_url': None,
720 }]
721
722
723 class GoogleIE(InfoExtractor):
724 """Information extractor for video.google.com."""
725
726 _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
727 IE_NAME = u'video.google'
728
729 def __init__(self, downloader=None):
730 InfoExtractor.__init__(self, downloader)
731
732 def report_download_webpage(self, video_id):
733 """Report webpage download."""
734 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
735
736 def report_extraction(self, video_id):
737 """Report information extraction."""
738 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
739
740 def _real_extract(self, url):
741 # Extract id from URL
742 mobj = re.match(self._VALID_URL, url)
743 if mobj is None:
744 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
745 return
746
747 video_id = mobj.group(1)
748
749 video_extension = 'mp4'
750
751 # Retrieve video webpage to extract further information
752 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
753 try:
754 self.report_download_webpage(video_id)
755 webpage = urllib2.urlopen(request).read()
756 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
757 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
758 return
759
760 # Extract URL, uploader, and title from webpage
761 self.report_extraction(video_id)
762 mobj = re.search(r"download_url:'([^']+)'", webpage)
763 if mobj is None:
764 video_extension = 'flv'
765 mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
766 if mobj is None:
767 self._downloader.trouble(u'ERROR: unable to extract media URL')
768 return
769 mediaURL = urllib.unquote(mobj.group(1))
770 mediaURL = mediaURL.replace('\\x3d', '\x3d')
771 mediaURL = mediaURL.replace('\\x26', '\x26')
772
773 video_url = mediaURL
774
775 mobj = re.search(r'<title>(.*)</title>', webpage)
776 if mobj is None:
777 self._downloader.trouble(u'ERROR: unable to extract title')
778 return
779 video_title = mobj.group(1).decode('utf-8')
780
781 # Extract video description
782 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
783 if mobj is None:
784 self._downloader.trouble(u'ERROR: unable to extract video description')
785 return
786 video_description = mobj.group(1).decode('utf-8')
787 if not video_description:
788 video_description = 'No description available.'
789
790 # Extract video thumbnail
791 if self._downloader.params.get('forcethumbnail', False):
792 request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
793 try:
794 webpage = urllib2.urlopen(request).read()
795 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
796 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
797 return
798 mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
799 if mobj is None:
800 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
801 return
802 video_thumbnail = mobj.group(1)
803 else: # we need something to pass to process_info
804 video_thumbnail = ''
805
806 return [{
807 'id': video_id.decode('utf-8'),
808 'url': video_url.decode('utf-8'),
809 'uploader': u'NA',
810 'upload_date': u'NA',
811 'title': video_title,
812 'ext': video_extension.decode('utf-8'),
813 'format': u'NA',
814 'player_url': None,
815 }]
816
817
818 class PhotobucketIE(InfoExtractor):
819 """Information extractor for photobucket.com."""
820
821 _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
822 IE_NAME = u'photobucket'
823
824 def __init__(self, downloader=None):
825 InfoExtractor.__init__(self, downloader)
826
827 def report_download_webpage(self, video_id):
828 """Report webpage download."""
829 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
830
831 def report_extraction(self, video_id):
832 """Report information extraction."""
833 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
834
835 def _real_extract(self, url):
836 # Extract id from URL
837 mobj = re.match(self._VALID_URL, url)
838 if mobj is None:
839 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
840 return
841
842 video_id = mobj.group(1)
843
844 video_extension = 'flv'
845
846 # Retrieve video webpage to extract further information
847 request = urllib2.Request(url)
848 try:
849 self.report_download_webpage(video_id)
850 webpage = urllib2.urlopen(request).read()
851 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
852 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
853 return
854
855 # Extract URL, uploader, and title from webpage
856 self.report_extraction(video_id)
857 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
858 if mobj is None:
859 self._downloader.trouble(u'ERROR: unable to extract media URL')
860 return
861 mediaURL = urllib.unquote(mobj.group(1))
862
863 video_url = mediaURL
864
865 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
866 if mobj is None:
867 self._downloader.trouble(u'ERROR: unable to extract title')
868 return
869 video_title = mobj.group(1).decode('utf-8')
870
871 video_uploader = mobj.group(2).decode('utf-8')
872
873 return [{
874 'id': video_id.decode('utf-8'),
875 'url': video_url.decode('utf-8'),
876 'uploader': video_uploader,
877 'upload_date': u'NA',
878 'title': video_title,
879 'ext': video_extension.decode('utf-8'),
880 'format': u'NA',
881 'player_url': None,
882 }]
883
884
885 class YahooIE(InfoExtractor):
886 """Information extractor for video.yahoo.com."""
887
888 # _VALID_URL matches all Yahoo! Video URLs
889 # _VPAGE_URL matches only the extractable '/watch/' URLs
890 _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
891 _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
892 IE_NAME = u'video.yahoo'
893
894 def __init__(self, downloader=None):
895 InfoExtractor.__init__(self, downloader)
896
897 def report_download_webpage(self, video_id):
898 """Report webpage download."""
899 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
900
901 def report_extraction(self, video_id):
902 """Report information extraction."""
903 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
904
905 def _real_extract(self, url, new_video=True):
906 # Extract ID from URL
907 mobj = re.match(self._VALID_URL, url)
908 if mobj is None:
909 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
910 return
911
912 video_id = mobj.group(2)
913 video_extension = 'flv'
914
915 # Rewrite valid but non-extractable URLs as
916 # extractable English language /watch/ URLs
917 if re.match(self._VPAGE_URL, url) is None:
918 request = urllib2.Request(url)
919 try:
920 webpage = urllib2.urlopen(request).read()
921 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
922 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
923 return
924
925 mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
926 if mobj is None:
927 self._downloader.trouble(u'ERROR: Unable to extract id field')
928 return
929 yahoo_id = mobj.group(1)
930
931 mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
932 if mobj is None:
933 self._downloader.trouble(u'ERROR: Unable to extract vid field')
934 return
935 yahoo_vid = mobj.group(1)
936
937 url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
938 return self._real_extract(url, new_video=False)
939
940 # Retrieve video webpage to extract further information
941 request = urllib2.Request(url)
942 try:
943 self.report_download_webpage(video_id)
944 webpage = urllib2.urlopen(request).read()
945 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
946 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
947 return
948
949 # Extract uploader and title from webpage
950 self.report_extraction(video_id)
951 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
952 if mobj is None:
953 self._downloader.trouble(u'ERROR: unable to extract video title')
954 return
955 video_title = mobj.group(1).decode('utf-8')
956
957 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
958 if mobj is None:
959 self._downloader.trouble(u'ERROR: unable to extract video uploader')
960 return
961 video_uploader = mobj.group(1).decode('utf-8')
962
963 # Extract video thumbnail
964 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
965 if mobj is None:
966 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
967 return
968 video_thumbnail = mobj.group(1).decode('utf-8')
969
970 # Extract video description
971 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
972 if mobj is None:
973 self._downloader.trouble(u'ERROR: unable to extract video description')
974 return
975 video_description = mobj.group(1).decode('utf-8')
976 if not video_description:
977 video_description = 'No description available.'
978
979 # Extract video height and width
980 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
981 if mobj is None:
982 self._downloader.trouble(u'ERROR: unable to extract video height')
983 return
984 yv_video_height = mobj.group(1)
985
986 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
987 if mobj is None:
988 self._downloader.trouble(u'ERROR: unable to extract video width')
989 return
990 yv_video_width = mobj.group(1)
991
992 # Retrieve video playlist to extract media URL
993 # I'm not completely sure what all these options are, but we
994 # seem to need most of them, otherwise the server sends a 401.
995 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
996 yv_bitrate = '700' # according to Wikipedia this is hard-coded
997 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
998 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
999 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1000 try:
1001 self.report_download_webpage(video_id)
1002 webpage = urllib2.urlopen(request).read()
1003 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1004 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1005 return
1006
1007 # Extract media URL from playlist XML
1008 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1009 if mobj is None:
1010 self._downloader.trouble(u'ERROR: Unable to extract media URL')
1011 return
1012 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1013 video_url = unescapeHTML(video_url)
1014
1015 return [{
1016 'id': video_id.decode('utf-8'),
1017 'url': video_url,
1018 'uploader': video_uploader,
1019 'upload_date': u'NA',
1020 'title': video_title,
1021 'ext': video_extension.decode('utf-8'),
1022 'thumbnail': video_thumbnail.decode('utf-8'),
1023 'description': video_description,
1024 'thumbnail': video_thumbnail,
1025 'player_url': None,
1026 }]
1027
1028
1029 class VimeoIE(InfoExtractor):
1030 """Information extractor for vimeo.com."""
1031
1032 # _VALID_URL matches Vimeo URLs
1033 _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?:videos?/)?([0-9]+)'
1034 IE_NAME = u'vimeo'
1035
1036 def __init__(self, downloader=None):
1037 InfoExtractor.__init__(self, downloader)
1038
1039 def report_download_webpage(self, video_id):
1040 """Report webpage download."""
1041 self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1042
1043 def report_extraction(self, video_id):
1044 """Report information extraction."""
1045 self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1046
1047 def _real_extract(self, url, new_video=True):
1048 # Extract ID from URL
1049 mobj = re.match(self._VALID_URL, url)
1050 if mobj is None:
1051 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1052 return
1053
1054 video_id = mobj.group(1)
1055
1056 # Retrieve video webpage to extract further information
1057 request = urllib2.Request(url, None, std_headers)
1058 try:
1059 self.report_download_webpage(video_id)
1060 webpage = urllib2.urlopen(request).read()
1061 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1062 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1063 return
1064
1065 # Now we begin extracting as much information as we can from what we
1066 # retrieved. First we extract the information common to all extractors,
1067 # and latter we extract those that are Vimeo specific.
1068 self.report_extraction(video_id)
1069
1070 # Extract the config JSON
1071 config = webpage.split(' = {config:')[1].split(',assets:')[0]
1072 try:
1073 config = json.loads(config)
1074 except:
1075 self._downloader.trouble(u'ERROR: unable to extract info section')
1076 return
1077
1078 # Extract title
1079 video_title = config["video"]["title"]
1080
1081 # Extract uploader
1082 video_uploader = config["video"]["owner"]["name"]
1083
1084 # Extract video thumbnail
1085 video_thumbnail = config["video"]["thumbnail"]
1086
1087 # Extract video description
1088 video_description = get_element_by_id("description", webpage.decode('utf8'))
1089 if video_description: video_description = clean_html(video_description)
1090 else: video_description = ''
1091
1092 # Extract upload date
1093 video_upload_date = u'NA'
1094 mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
1095 if mobj is not None:
1096 video_upload_date = mobj.group(1)
1097
1098 # Vimeo specific: extract request signature and timestamp
1099 sig = config['request']['signature']
1100 timestamp = config['request']['timestamp']
1101
1102 # Vimeo specific: extract video codec and quality information
1103 # First consider quality, then codecs, then take everything
1104 # TODO bind to format param
1105 codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
1106 files = { 'hd': [], 'sd': [], 'other': []}
1107 for codec_name, codec_extension in codecs:
1108 if codec_name in config["video"]["files"]:
1109 if 'hd' in config["video"]["files"][codec_name]:
1110 files['hd'].append((codec_name, codec_extension, 'hd'))
1111 elif 'sd' in config["video"]["files"][codec_name]:
1112 files['sd'].append((codec_name, codec_extension, 'sd'))
1113 else:
1114 files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0]))
1115
1116 for quality in ('hd', 'sd', 'other'):
1117 if len(files[quality]) > 0:
1118 video_quality = files[quality][0][2]
1119 video_codec = files[quality][0][0]
1120 video_extension = files[quality][0][1]
1121 self._downloader.to_screen(u'[vimeo] %s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality))
1122 break
1123 else:
1124 self._downloader.trouble(u'ERROR: no known codec found')
1125 return
1126
1127 video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
1128 %(video_id, sig, timestamp, video_quality, video_codec.upper())
1129
1130 return [{
1131 'id': video_id,
1132 'url': video_url,
1133 'uploader': video_uploader,
1134 'upload_date': video_upload_date,
1135 'title': video_title,
1136 'ext': video_extension,
1137 'thumbnail': video_thumbnail,
1138 'description': video_description,
1139 'player_url': None,
1140 }]
1141
1142
1143 class GenericIE(InfoExtractor):
1144 """Generic last-resort information extractor."""
1145
1146 _VALID_URL = r'.*'
1147 IE_NAME = u'generic'
1148
1149 def __init__(self, downloader=None):
1150 InfoExtractor.__init__(self, downloader)
1151
1152 def report_download_webpage(self, video_id):
1153 """Report webpage download."""
1154 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1155 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1156
1157 def report_extraction(self, video_id):
1158 """Report information extraction."""
1159 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1160
1161 def report_following_redirect(self, new_url):
1162 """Report information extraction."""
1163 self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
1164
1165 def _test_redirect(self, url):
1166 """Check if it is a redirect, like url shorteners, in case restart chain."""
1167 class HeadRequest(urllib2.Request):
1168 def get_method(self):
1169 return "HEAD"
1170
1171 class HEADRedirectHandler(urllib2.HTTPRedirectHandler):
1172 """
1173 Subclass the HTTPRedirectHandler to make it use our
1174 HeadRequest also on the redirected URL
1175 """
1176 def redirect_request(self, req, fp, code, msg, headers, newurl):
1177 if code in (301, 302, 303, 307):
1178 newurl = newurl.replace(' ', '%20')
1179 newheaders = dict((k,v) for k,v in req.headers.items()
1180 if k.lower() not in ("content-length", "content-type"))
1181 return HeadRequest(newurl,
1182 headers=newheaders,
1183 origin_req_host=req.get_origin_req_host(),
1184 unverifiable=True)
1185 else:
1186 raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
1187
1188 class HTTPMethodFallback(urllib2.BaseHandler):
1189 """
1190 Fallback to GET if HEAD is not allowed (405 HTTP error)
1191 """
1192 def http_error_405(self, req, fp, code, msg, headers):
1193 fp.read()
1194 fp.close()
1195
1196 newheaders = dict((k,v) for k,v in req.headers.items()
1197 if k.lower() not in ("content-length", "content-type"))
1198 return self.parent.open(urllib2.Request(req.get_full_url(),
1199 headers=newheaders,
1200 origin_req_host=req.get_origin_req_host(),
1201 unverifiable=True))
1202
1203 # Build our opener
1204 opener = urllib2.OpenerDirector()
1205 for handler in [urllib2.HTTPHandler, urllib2.HTTPDefaultErrorHandler,
1206 HTTPMethodFallback, HEADRedirectHandler,
1207 urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler]:
1208 opener.add_handler(handler())
1209
1210 response = opener.open(HeadRequest(url))
1211 new_url = response.geturl()
1212
1213 if url == new_url: return False
1214
1215 self.report_following_redirect(new_url)
1216 self._downloader.download([new_url])
1217 return True
1218
1219 def _real_extract(self, url):
1220 if self._test_redirect(url): return
1221
1222 video_id = url.split('/')[-1]
1223 request = urllib2.Request(url)
1224 try:
1225 self.report_download_webpage(video_id)
1226 webpage = urllib2.urlopen(request).read()
1227 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1228 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1229 return
1230 except ValueError, err:
1231 # since this is the last-resort InfoExtractor, if
1232 # this error is thrown, it'll be thrown here
1233 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1234 return
1235
1236 self.report_extraction(video_id)
1237 # Start with something easy: JW Player in SWFObject
1238 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1239 if mobj is None:
1240 # Broaden the search a little bit
1241 mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1242 if mobj is None:
1243 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1244 return
1245
1246 # It's possible that one of the regexes
1247 # matched, but returned an empty group:
1248 if mobj.group(1) is None:
1249 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1250 return
1251
1252 video_url = urllib.unquote(mobj.group(1))
1253 video_id = os.path.basename(video_url)
1254
1255 # here's a fun little line of code for you:
1256 video_extension = os.path.splitext(video_id)[1][1:]
1257 video_id = os.path.splitext(video_id)[0]
1258
1259 # it's tempting to parse this further, but you would
1260 # have to take into account all the variations like
1261 # Video Title - Site Name
1262 # Site Name | Video Title
1263 # Video Title - Tagline | Site Name
1264 # and so on and so forth; it's just not practical
1265 mobj = re.search(r'<title>(.*)</title>', webpage)
1266 if mobj is None:
1267 self._downloader.trouble(u'ERROR: unable to extract title')
1268 return
1269 video_title = mobj.group(1).decode('utf-8')
1270
1271 # video uploader is domain name
1272 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1273 if mobj is None:
1274 self._downloader.trouble(u'ERROR: unable to extract title')
1275 return
1276 video_uploader = mobj.group(1).decode('utf-8')
1277
1278 return [{
1279 'id': video_id.decode('utf-8'),
1280 'url': video_url.decode('utf-8'),
1281 'uploader': video_uploader,
1282 'upload_date': u'NA',
1283 'title': video_title,
1284 'ext': video_extension.decode('utf-8'),
1285 'format': u'NA',
1286 'player_url': None,
1287 }]
1288
1289
1290 class YoutubeSearchIE(InfoExtractor):
1291 """Information Extractor for YouTube search queries."""
1292 _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
1293 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1294 _max_youtube_results = 1000
1295 IE_NAME = u'youtube:search'
1296
1297 def __init__(self, downloader=None):
1298 InfoExtractor.__init__(self, downloader)
1299
1300 def report_download_page(self, query, pagenum):
1301 """Report attempt to download search page with given number."""
1302 query = query.decode(preferredencoding())
1303 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1304
1305 def _real_extract(self, query):
1306 mobj = re.match(self._VALID_URL, query)
1307 if mobj is None:
1308 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1309 return
1310
1311 prefix, query = query.split(':')
1312 prefix = prefix[8:]
1313 query = query.encode('utf-8')
1314 if prefix == '':
1315 self._download_n_results(query, 1)
1316 return
1317 elif prefix == 'all':
1318 self._download_n_results(query, self._max_youtube_results)
1319 return
1320 else:
1321 try:
1322 n = long(prefix)
1323 if n <= 0:
1324 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1325 return
1326 elif n > self._max_youtube_results:
1327 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1328 n = self._max_youtube_results
1329 self._download_n_results(query, n)
1330 return
1331 except ValueError: # parsing prefix as integer fails
1332 self._download_n_results(query, 1)
1333 return
1334
1335 def _download_n_results(self, query, n):
1336 """Downloads a specified number of results for a query"""
1337
1338 video_ids = []
1339 pagenum = 0
1340 limit = n
1341
1342 while (50 * pagenum) < limit:
1343 self.report_download_page(query, pagenum+1)
1344 result_url = self._API_URL % (urllib.quote_plus(query), (50*pagenum)+1)
1345 request = urllib2.Request(result_url)
1346 try:
1347 data = urllib2.urlopen(request).read()
1348 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1349 self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
1350 return
1351 api_response = json.loads(data)['data']
1352
1353 new_ids = list(video['id'] for video in api_response['items'])
1354 video_ids += new_ids
1355
1356 limit = min(n, api_response['totalItems'])
1357 pagenum += 1
1358
1359 if len(video_ids) > n:
1360 video_ids = video_ids[:n]
1361 for id in video_ids:
1362 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1363 return
1364
1365
1366 class GoogleSearchIE(InfoExtractor):
1367 """Information Extractor for Google Video search queries."""
1368 _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
1369 _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1370 _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
1371 _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
1372 _max_google_results = 1000
1373 IE_NAME = u'video.google:search'
1374
1375 def __init__(self, downloader=None):
1376 InfoExtractor.__init__(self, downloader)
1377
1378 def report_download_page(self, query, pagenum):
1379 """Report attempt to download playlist page with given number."""
1380 query = query.decode(preferredencoding())
1381 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1382
1383 def _real_extract(self, query):
1384 mobj = re.match(self._VALID_URL, query)
1385 if mobj is None:
1386 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1387 return
1388
1389 prefix, query = query.split(':')
1390 prefix = prefix[8:]
1391 query = query.encode('utf-8')
1392 if prefix == '':
1393 self._download_n_results(query, 1)
1394 return
1395 elif prefix == 'all':
1396 self._download_n_results(query, self._max_google_results)
1397 return
1398 else:
1399 try:
1400 n = long(prefix)
1401 if n <= 0:
1402 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1403 return
1404 elif n > self._max_google_results:
1405 self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1406 n = self._max_google_results
1407 self._download_n_results(query, n)
1408 return
1409 except ValueError: # parsing prefix as integer fails
1410 self._download_n_results(query, 1)
1411 return
1412
1413 def _download_n_results(self, query, n):
1414 """Downloads a specified number of results for a query"""
1415
1416 video_ids = []
1417 pagenum = 0
1418
1419 while True:
1420 self.report_download_page(query, pagenum)
1421 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum*10)
1422 request = urllib2.Request(result_url)
1423 try:
1424 page = urllib2.urlopen(request).read()
1425 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1426 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1427 return
1428
1429 # Extract video identifiers
1430 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1431 video_id = mobj.group(1)
1432 if video_id not in video_ids:
1433 video_ids.append(video_id)
1434 if len(video_ids) == n:
1435 # Specified n videos reached
1436 for id in video_ids:
1437 self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1438 return
1439
1440 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1441 for id in video_ids:
1442 self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1443 return
1444
1445 pagenum = pagenum + 1
1446
1447
1448 class YahooSearchIE(InfoExtractor):
1449 """Information Extractor for Yahoo! Video search queries."""
1450 _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
1451 _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1452 _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1453 _MORE_PAGES_INDICATOR = r'\s*Next'
1454 _max_yahoo_results = 1000
1455 IE_NAME = u'video.yahoo:search'
1456
1457 def __init__(self, downloader=None):
1458 InfoExtractor.__init__(self, downloader)
1459
1460 def report_download_page(self, query, pagenum):
1461 """Report attempt to download playlist page with given number."""
1462 query = query.decode(preferredencoding())
1463 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1464
1465 def _real_extract(self, query):
1466 mobj = re.match(self._VALID_URL, query)
1467 if mobj is None:
1468 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1469 return
1470
1471 prefix, query = query.split(':')
1472 prefix = prefix[8:]
1473 query = query.encode('utf-8')
1474 if prefix == '':
1475 self._download_n_results(query, 1)
1476 return
1477 elif prefix == 'all':
1478 self._download_n_results(query, self._max_yahoo_results)
1479 return
1480 else:
1481 try:
1482 n = long(prefix)
1483 if n <= 0:
1484 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1485 return
1486 elif n > self._max_yahoo_results:
1487 self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1488 n = self._max_yahoo_results
1489 self._download_n_results(query, n)
1490 return
1491 except ValueError: # parsing prefix as integer fails
1492 self._download_n_results(query, 1)
1493 return
1494
1495 def _download_n_results(self, query, n):
1496 """Downloads a specified number of results for a query"""
1497
1498 video_ids = []
1499 already_seen = set()
1500 pagenum = 1
1501
1502 while True:
1503 self.report_download_page(query, pagenum)
1504 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1505 request = urllib2.Request(result_url)
1506 try:
1507 page = urllib2.urlopen(request).read()
1508 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1509 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1510 return
1511
1512 # Extract video identifiers
1513 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1514 video_id = mobj.group(1)
1515 if video_id not in already_seen:
1516 video_ids.append(video_id)
1517 already_seen.add(video_id)
1518 if len(video_ids) == n:
1519 # Specified n videos reached
1520 for id in video_ids:
1521 self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1522 return
1523
1524 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1525 for id in video_ids:
1526 self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1527 return
1528
1529 pagenum = pagenum + 1
1530
1531
1532 class YoutubePlaylistIE(InfoExtractor):
1533 """Information Extractor for YouTube playlists."""
1534
1535 _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
1536 _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
1537 _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;([^&"]+&amp;)*list=.*?%s'
1538 _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
1539 IE_NAME = u'youtube:playlist'
1540
1541 def __init__(self, downloader=None):
1542 InfoExtractor.__init__(self, downloader)
1543
1544 def report_download_page(self, playlist_id, pagenum):
1545 """Report attempt to download playlist page with given number."""
1546 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1547
1548 def _real_extract(self, url):
1549 # Extract playlist id
1550 mobj = re.match(self._VALID_URL, url)
1551 if mobj is None:
1552 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1553 return
1554
1555 # Single video case
1556 if mobj.group(3) is not None:
1557 self._downloader.download([mobj.group(3)])
1558 return
1559
1560 # Download playlist pages
1561 # prefix is 'p' as default for playlists but there are other types that need extra care
1562 playlist_prefix = mobj.group(1)
1563 if playlist_prefix == 'a':
1564 playlist_access = 'artist'
1565 else:
1566 playlist_prefix = 'p'
1567 playlist_access = 'view_play_list'
1568 playlist_id = mobj.group(2)
1569 video_ids = []
1570 pagenum = 1
1571
1572 while True:
1573 self.report_download_page(playlist_id, pagenum)
1574 url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
1575 request = urllib2.Request(url)
1576 try:
1577 page = urllib2.urlopen(request).read()
1578 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1579 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1580 return
1581
1582 # Extract video identifiers
1583 ids_in_page = []
1584 for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page):
1585 if mobj.group(1) not in ids_in_page:
1586 ids_in_page.append(mobj.group(1))
1587 video_ids.extend(ids_in_page)
1588
1589 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1590 break
1591 pagenum = pagenum + 1
1592
1593 playliststart = self._downloader.params.get('playliststart', 1) - 1
1594 playlistend = self._downloader.params.get('playlistend', -1)
1595 if playlistend == -1:
1596 video_ids = video_ids[playliststart:]
1597 else:
1598 video_ids = video_ids[playliststart:playlistend]
1599
1600 for id in video_ids:
1601 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1602 return
1603
1604
1605 class YoutubeChannelIE(InfoExtractor):
1606 """Information Extractor for YouTube channels."""
1607
1608 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
1609 _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
1610 _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO
1611 IE_NAME = u'youtube:channel'
1612
1613 def report_download_page(self, channel_id, pagenum):
1614 """Report attempt to download channel page with given number."""
1615 self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
1616
1617 def _real_extract(self, url):
1618 # Extract channel id
1619 mobj = re.match(self._VALID_URL, url)
1620 if mobj is None:
1621 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1622 return
1623
1624 # Download channel pages
1625 channel_id = mobj.group(1)
1626 video_ids = []
1627 pagenum = 1
1628
1629 while True:
1630 self.report_download_page(channel_id, pagenum)
1631 url = self._TEMPLATE_URL % (channel_id, pagenum)
1632 request = urllib2.Request(url)
1633 try:
1634 page = urllib2.urlopen(request).read()
1635 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1636 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1637 return
1638
1639 # Extract video identifiers
1640 ids_in_page = []
1641 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
1642 if mobj.group(1) not in ids_in_page:
1643 ids_in_page.append(mobj.group(1))
1644 video_ids.extend(ids_in_page)
1645
1646 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1647 break
1648 pagenum = pagenum + 1
1649
1650 for id in video_ids:
1651 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1652 return
1653
1654
1655 class YoutubeUserIE(InfoExtractor):
1656 """Information Extractor for YouTube users."""
1657
1658 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
1659 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1660 _GDATA_PAGE_SIZE = 50
1661 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
1662 _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
1663 IE_NAME = u'youtube:user'
1664
1665 def __init__(self, downloader=None):
1666 InfoExtractor.__init__(self, downloader)
1667
1668 def report_download_page(self, username, start_index):
1669 """Report attempt to download user page."""
1670 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
1671 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
1672
1673 def _real_extract(self, url):
1674 # Extract username
1675 mobj = re.match(self._VALID_URL, url)
1676 if mobj is None:
1677 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1678 return
1679
1680 username = mobj.group(1)
1681
1682 # Download video ids using YouTube Data API. Result size per
1683 # query is limited (currently to 50 videos) so we need to query
1684 # page by page until there are no video ids - it means we got
1685 # all of them.
1686
1687 video_ids = []
1688 pagenum = 0
1689
1690 while True:
1691 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1692 self.report_download_page(username, start_index)
1693
1694 request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
1695
1696 try:
1697 page = urllib2.urlopen(request).read()
1698 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1699 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1700 return
1701
1702 # Extract video identifiers
1703 ids_in_page = []
1704
1705 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1706 if mobj.group(1) not in ids_in_page:
1707 ids_in_page.append(mobj.group(1))
1708
1709 video_ids.extend(ids_in_page)
1710
1711 # A little optimization - if current page is not
1712 # "full", ie. does not contain PAGE_SIZE video ids then
1713 # we can assume that this page is the last one - there
1714 # are no more ids on further pages - no need to query
1715 # again.
1716
1717 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1718 break
1719
1720 pagenum += 1
1721
1722 all_ids_count = len(video_ids)
1723 playliststart = self._downloader.params.get('playliststart', 1) - 1
1724 playlistend = self._downloader.params.get('playlistend', -1)
1725
1726 if playlistend == -1:
1727 video_ids = video_ids[playliststart:]
1728 else:
1729 video_ids = video_ids[playliststart:playlistend]
1730
1731 self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
1732 (username, all_ids_count, len(video_ids)))
1733
1734 for video_id in video_ids:
1735 self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
1736
1737
1738 class BlipTVUserIE(InfoExtractor):
1739 """Information Extractor for blip.tv users."""
1740
1741 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
1742 _PAGE_SIZE = 12
1743 IE_NAME = u'blip.tv:user'
1744
1745 def __init__(self, downloader=None):
1746 InfoExtractor.__init__(self, downloader)
1747
1748 def report_download_page(self, username, pagenum):
1749 """Report attempt to download user page."""
1750 self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
1751 (self.IE_NAME, username, pagenum))
1752
1753 def _real_extract(self, url):
1754 # Extract username
1755 mobj = re.match(self._VALID_URL, url)
1756 if mobj is None:
1757 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1758 return
1759
1760 username = mobj.group(1)
1761
1762 page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
1763
1764 request = urllib2.Request(url)
1765
1766 try:
1767 page = urllib2.urlopen(request).read().decode('utf-8')
1768 mobj = re.search(r'data-users-id="([^"]+)"', page)
1769 page_base = page_base % mobj.group(1)
1770 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1771 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1772 return
1773
1774
1775 # Download video ids using BlipTV Ajax calls. Result size per
1776 # query is limited (currently to 12 videos) so we need to query
1777 # page by page until there are no video ids - it means we got
1778 # all of them.
1779
1780 video_ids = []
1781 pagenum = 1
1782
1783 while True:
1784 self.report_download_page(username, pagenum)
1785
1786 request = urllib2.Request( page_base + "&page=" + str(pagenum) )
1787
1788 try:
1789 page = urllib2.urlopen(request).read().decode('utf-8')
1790 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1791 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1792 return
1793
1794 # Extract video identifiers
1795 ids_in_page = []
1796
1797 for mobj in re.finditer(r'href="/([^"]+)"', page):
1798 if mobj.group(1) not in ids_in_page:
1799 ids_in_page.append(unescapeHTML(mobj.group(1)))
1800
1801 video_ids.extend(ids_in_page)
1802
1803 # A little optimization - if current page is not
1804 # "full", ie. does not contain PAGE_SIZE video ids then
1805 # we can assume that this page is the last one - there
1806 # are no more ids on further pages - no need to query
1807 # again.
1808
1809 if len(ids_in_page) < self._PAGE_SIZE:
1810 break
1811
1812 pagenum += 1
1813
1814 all_ids_count = len(video_ids)
1815 playliststart = self._downloader.params.get('playliststart', 1) - 1
1816 playlistend = self._downloader.params.get('playlistend', -1)
1817
1818 if playlistend == -1:
1819 video_ids = video_ids[playliststart:]
1820 else:
1821 video_ids = video_ids[playliststart:playlistend]
1822
1823 self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
1824 (self.IE_NAME, username, all_ids_count, len(video_ids)))
1825
1826 for video_id in video_ids:
1827 self._downloader.download([u'http://blip.tv/'+video_id])
1828
1829
1830 class DepositFilesIE(InfoExtractor):
1831 """Information extractor for depositfiles.com"""
1832
1833 _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
1834 IE_NAME = u'DepositFiles'
1835
1836 def __init__(self, downloader=None):
1837 InfoExtractor.__init__(self, downloader)
1838
1839 def report_download_webpage(self, file_id):
1840 """Report webpage download."""
1841 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
1842
1843 def report_extraction(self, file_id):
1844 """Report information extraction."""
1845 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
1846
1847 def _real_extract(self, url):
1848 file_id = url.split('/')[-1]
1849 # Rebuild url in english locale
1850 url = 'http://depositfiles.com/en/files/' + file_id
1851
1852 # Retrieve file webpage with 'Free download' button pressed
1853 free_download_indication = { 'gateway_result' : '1' }
1854 request = urllib2.Request(url, urllib.urlencode(free_download_indication))
1855 try:
1856 self.report_download_webpage(file_id)
1857 webpage = urllib2.urlopen(request).read()
1858 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1859 self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
1860 return
1861
1862 # Search for the real file URL
1863 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
1864 if (mobj is None) or (mobj.group(1) is None):
1865 # Try to figure out reason of the error.
1866 mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
1867 if (mobj is not None) and (mobj.group(1) is not None):
1868 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
1869 self._downloader.trouble(u'ERROR: %s' % restriction_message)
1870 else:
1871 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
1872 return
1873
1874 file_url = mobj.group(1)
1875 file_extension = os.path.splitext(file_url)[1][1:]
1876
1877 # Search for file title
1878 mobj = re.search(r'<b title="(.*?)">', webpage)
1879 if mobj is None:
1880 self._downloader.trouble(u'ERROR: unable to extract title')
1881 return
1882 file_title = mobj.group(1).decode('utf-8')
1883
1884 return [{
1885 'id': file_id.decode('utf-8'),
1886 'url': file_url.decode('utf-8'),
1887 'uploader': u'NA',
1888 'upload_date': u'NA',
1889 'title': file_title,
1890 'ext': file_extension.decode('utf-8'),
1891 'format': u'NA',
1892 'player_url': None,
1893 }]
1894
1895
1896 class FacebookIE(InfoExtractor):
1897 """Information Extractor for Facebook"""
1898
1899 _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
1900 _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
1901 _NETRC_MACHINE = 'facebook'
1902 _available_formats = ['video', 'highqual', 'lowqual']
1903 _video_extensions = {
1904 'video': 'mp4',
1905 'highqual': 'mp4',
1906 'lowqual': 'mp4',
1907 }
1908 IE_NAME = u'facebook'
1909
1910 def __init__(self, downloader=None):
1911 InfoExtractor.__init__(self, downloader)
1912
1913 def _reporter(self, message):
1914 """Add header and report message."""
1915 self._downloader.to_screen(u'[facebook] %s' % message)
1916
1917 def report_login(self):
1918 """Report attempt to log in."""
1919 self._reporter(u'Logging in')
1920
1921 def report_video_webpage_download(self, video_id):
1922 """Report attempt to download video webpage."""
1923 self._reporter(u'%s: Downloading video webpage' % video_id)
1924
1925 def report_information_extraction(self, video_id):
1926 """Report attempt to extract video information."""
1927 self._reporter(u'%s: Extracting video information' % video_id)
1928
1929 def _parse_page(self, video_webpage):
1930 """Extract video information from page"""
1931 # General data
1932 data = {'title': r'\("video_title", "(.*?)"\)',
1933 'description': r'<div class="datawrap">(.*?)</div>',
1934 'owner': r'\("video_owner_name", "(.*?)"\)',
1935 'thumbnail': r'\("thumb_url", "(?P<THUMB>.*?)"\)',
1936 }
1937 video_info = {}
1938 for piece in data.keys():
1939 mobj = re.search(data[piece], video_webpage)
1940 if mobj is not None:
1941 video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
1942
1943 # Video urls
1944 video_urls = {}
1945 for fmt in self._available_formats:
1946 mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
1947 if mobj is not None:
1948 # URL is in a Javascript segment inside an escaped Unicode format within
1949 # the generally utf-8 page
1950 video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
1951 video_info['video_urls'] = video_urls
1952
1953 return video_info
1954
1955 def _real_initialize(self):
1956 if self._downloader is None:
1957 return
1958
1959 useremail = None
1960 password = None
1961 downloader_params = self._downloader.params
1962
1963 # Attempt to use provided username and password or .netrc data
1964 if downloader_params.get('username', None) is not None:
1965 useremail = downloader_params['username']
1966 password = downloader_params['password']
1967 elif downloader_params.get('usenetrc', False):
1968 try:
1969 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
1970 if info is not None:
1971 useremail = info[0]
1972 password = info[2]
1973 else:
1974 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
1975 except (IOError, netrc.NetrcParseError), err:
1976 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
1977 return
1978
1979 if useremail is None:
1980 return
1981
1982 # Log in
1983 login_form = {
1984 'email': useremail,
1985 'pass': password,
1986 'login': 'Log+In'
1987 }
1988 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
1989 try:
1990 self.report_login()
1991 login_results = urllib2.urlopen(request).read()
1992 if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
1993 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
1994 return
1995 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1996 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
1997 return
1998
1999 def _real_extract(self, url):
2000 mobj = re.match(self._VALID_URL, url)
2001 if mobj is None:
2002 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2003 return
2004 video_id = mobj.group('ID')
2005
2006 # Get video webpage
2007 self.report_video_webpage_download(video_id)
2008 request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
2009 try:
2010 page = urllib2.urlopen(request)
2011 video_webpage = page.read()
2012 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2013 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2014 return
2015
2016 # Start extracting information
2017 self.report_information_extraction(video_id)
2018
2019 # Extract information
2020 video_info = self._parse_page(video_webpage)
2021
2022 # uploader
2023 if 'owner' not in video_info:
2024 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
2025 return
2026 video_uploader = video_info['owner']
2027
2028 # title
2029 if 'title' not in video_info:
2030 self._downloader.trouble(u'ERROR: unable to extract video title')
2031 return
2032 video_title = video_info['title']
2033 video_title = video_title.decode('utf-8')
2034
2035 # thumbnail image
2036 if 'thumbnail' not in video_info:
2037 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
2038 video_thumbnail = ''
2039 else:
2040 video_thumbnail = video_info['thumbnail']
2041
2042 # upload date
2043 upload_date = u'NA'
2044 if 'upload_date' in video_info:
2045 upload_time = video_info['upload_date']
2046 timetuple = email.utils.parsedate_tz(upload_time)
2047 if timetuple is not None:
2048 try:
2049 upload_date = time.strftime('%Y%m%d', timetuple[0:9])
2050 except:
2051 pass
2052
2053 # description
2054 video_description = video_info.get('description', 'No description available.')
2055
2056 url_map = video_info['video_urls']
2057 if len(url_map.keys()) > 0:
2058 # Decide which formats to download
2059 req_format = self._downloader.params.get('format', None)
2060 format_limit = self._downloader.params.get('format_limit', None)
2061
2062 if format_limit is not None and format_limit in self._available_formats:
2063 format_list = self._available_formats[self._available_formats.index(format_limit):]
2064 else:
2065 format_list = self._available_formats
2066 existing_formats = [x for x in format_list if x in url_map]
2067 if len(existing_formats) == 0:
2068 self._downloader.trouble(u'ERROR: no known formats available for video')
2069 return
2070 if req_format is None:
2071 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2072 elif req_format == 'worst':
2073 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
2074 elif req_format == '-1':
2075 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2076 else:
2077 # Specific format
2078 if req_format not in url_map:
2079 self._downloader.trouble(u'ERROR: requested format not available')
2080 return
2081 video_url_list = [(req_format, url_map[req_format])] # Specific format
2082
2083 results = []
2084 for format_param, video_real_url in video_url_list:
2085 # Extension
2086 video_extension = self._video_extensions.get(format_param, 'mp4')
2087
2088 results.append({
2089 'id': video_id.decode('utf-8'),
2090 'url': video_real_url.decode('utf-8'),
2091 'uploader': video_uploader.decode('utf-8'),
2092 'upload_date': upload_date,
2093 'title': video_title,
2094 'ext': video_extension.decode('utf-8'),
2095 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
2096 'thumbnail': video_thumbnail.decode('utf-8'),
2097 'description': video_description.decode('utf-8'),
2098 'player_url': None,
2099 })
2100 return results
2101
2102 class BlipTVIE(InfoExtractor):
2103 """Information extractor for blip.tv"""
2104
2105 _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2106 _URL_EXT = r'^.*\.([a-z0-9]+)$'
2107 IE_NAME = u'blip.tv'
2108
2109 def report_extraction(self, file_id):
2110 """Report information extraction."""
2111 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2112
2113 def report_direct_download(self, title):
2114 """Report information extraction."""
2115 self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
2116
2117 def _real_extract(self, url):
2118 mobj = re.match(self._VALID_URL, url)
2119 if mobj is None:
2120 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2121 return
2122
2123 if '?' in url:
2124 cchar = '&'
2125 else:
2126 cchar = '?'
2127 json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2128 request = urllib2.Request(json_url.encode('utf-8'))
2129 self.report_extraction(mobj.group(1))
2130 info = None
2131 try:
2132 urlh = urllib2.urlopen(request)
2133 if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
2134 basename = url.split('/')[-1]
2135 title,ext = os.path.splitext(basename)
2136 title = title.decode('UTF-8')
2137 ext = ext.replace('.', '')
2138 self.report_direct_download(title)
2139 info = {
2140 'id': title,
2141 'url': url,
2142 'title': title,
2143 'ext': ext,
2144 'urlhandle': urlh
2145 }
2146 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2147 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
2148 return
2149 if info is None: # Regular URL
2150 try:
2151 json_code = urlh.read()
2152 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2153 self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
2154 return
2155
2156 try:
2157 json_data = json.loads(json_code)
2158 if 'Post' in json_data:
2159 data = json_data['Post']
2160 else:
2161 data = json_data
2162
2163 upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2164 video_url = data['media']['url']
2165 umobj = re.match(self._URL_EXT, video_url)
2166 if umobj is None:
2167 raise ValueError('Can not determine filename extension')
2168 ext = umobj.group(1)
2169
2170 info = {
2171 'id': data['item_id'],
2172 'url': video_url,
2173 'uploader': data['display_name'],
2174 'upload_date': upload_date,
2175 'title': data['title'],
2176 'ext': ext,
2177 'format': data['media']['mimeType'],
2178 'thumbnail': data['thumbnailUrl'],
2179 'description': data['description'],
2180 'player_url': data['embedUrl']
2181 }
2182 except (ValueError,KeyError), err:
2183 self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2184 return
2185
2186 std_headers['User-Agent'] = 'iTunes/10.6.1'
2187 return [info]
2188
2189
2190 class MyVideoIE(InfoExtractor):
2191 """Information Extractor for myvideo.de."""
2192
2193 _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
2194 IE_NAME = u'myvideo'
2195
2196 def __init__(self, downloader=None):
2197 InfoExtractor.__init__(self, downloader)
2198
2199 def report_download_webpage(self, video_id):
2200 """Report webpage download."""
2201 self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
2202
2203 def report_extraction(self, video_id):
2204 """Report information extraction."""
2205 self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
2206
2207 def _real_extract(self,url):
2208 mobj = re.match(self._VALID_URL, url)
2209 if mobj is None:
2210 self._download.trouble(u'ERROR: invalid URL: %s' % url)
2211 return
2212
2213 video_id = mobj.group(1)
2214
2215 # Get video webpage
2216 request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
2217 try:
2218 self.report_download_webpage(video_id)
2219 webpage = urllib2.urlopen(request).read()
2220 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2221 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
2222 return
2223
2224 self.report_extraction(video_id)
2225 mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
2226 webpage)
2227 if mobj is None:
2228 self._downloader.trouble(u'ERROR: unable to extract media URL')
2229 return
2230 video_url = mobj.group(1) + ('/%s.flv' % video_id)
2231
2232 mobj = re.search('<title>([^<]+)</title>', webpage)
2233 if mobj is None:
2234 self._downloader.trouble(u'ERROR: unable to extract title')
2235 return
2236
2237 video_title = mobj.group(1)
2238
2239 return [{
2240 'id': video_id,
2241 'url': video_url,
2242 'uploader': u'NA',
2243 'upload_date': u'NA',
2244 'title': video_title,
2245 'ext': u'flv',
2246 'format': u'NA',
2247 'player_url': None,
2248 }]
2249
2250 class ComedyCentralIE(InfoExtractor):
2251 """Information extractor for The Daily Show and Colbert Report """
2252
2253 _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
2254 IE_NAME = u'comedycentral'
2255
2256 _available_formats = ['3500', '2200', '1700', '1200', '750', '400']
2257
2258 _video_extensions = {
2259 '3500': 'mp4',
2260 '2200': 'mp4',
2261 '1700': 'mp4',
2262 '1200': 'mp4',
2263 '750': 'mp4',
2264 '400': 'mp4',
2265 }
2266 _video_dimensions = {
2267 '3500': '1280x720',
2268 '2200': '960x540',
2269 '1700': '768x432',
2270 '1200': '640x360',
2271 '750': '512x288',
2272 '400': '384x216',
2273 }
2274
2275 def report_extraction(self, episode_id):
2276 self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
2277
2278 def report_config_download(self, episode_id):
2279 self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
2280
2281 def report_index_download(self, episode_id):
2282 self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
2283
2284 def report_player_url(self, episode_id):
2285 self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
2286
2287
2288 def _print_formats(self, formats):
2289 print('Available formats:')
2290 for x in formats:
2291 print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
2292
2293
2294
2295 def _real_extract(self, url):
2296 mobj = re.match(self._VALID_URL, url)
2297 if mobj is None:
2298 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2299 return
2300
2301 if mobj.group('shortname'):
2302 if mobj.group('shortname') in ('tds', 'thedailyshow'):
2303 url = u'http://www.thedailyshow.com/full-episodes/'
2304 else:
2305 url = u'http://www.colbertnation.com/full-episodes/'
2306 mobj = re.match(self._VALID_URL, url)
2307 assert mobj is not None
2308
2309 dlNewest = not mobj.group('episode')
2310 if dlNewest:
2311 epTitle = mobj.group('showname')
2312 else:
2313 epTitle = mobj.group('episode')
2314
2315 req = urllib2.Request(url)
2316 self.report_extraction(epTitle)
2317 try:
2318 htmlHandle = urllib2.urlopen(req)
2319 html = htmlHandle.read()
2320 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2321 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
2322 return
2323 if dlNewest:
2324 url = htmlHandle.geturl()
2325 mobj = re.match(self._VALID_URL, url)
2326 if mobj is None:
2327 self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
2328 return
2329 if mobj.group('episode') == '':
2330 self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
2331 return
2332 epTitle = mobj.group('episode')
2333
2334 mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
2335 if len(mMovieParams) == 0:
2336 self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
2337 return
2338
2339 playerUrl_raw = mMovieParams[0][0]
2340 self.report_player_url(epTitle)
2341 try:
2342 urlHandle = urllib2.urlopen(playerUrl_raw)
2343 playerUrl = urlHandle.geturl()
2344 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2345 self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
2346 return
2347
2348 uri = mMovieParams[0][1]
2349 indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri})
2350 self.report_index_download(epTitle)
2351 try:
2352 indexXml = urllib2.urlopen(indexUrl).read()
2353 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2354 self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
2355 return
2356
2357 results = []
2358
2359 idoc = xml.etree.ElementTree.fromstring(indexXml)
2360 itemEls = idoc.findall('.//item')
2361 for itemEl in itemEls:
2362 mediaId = itemEl.findall('./guid')[0].text
2363 shortMediaId = mediaId.split(':')[-1]
2364 showId = mediaId.split(':')[-2].replace('.com', '')
2365 officialTitle = itemEl.findall('./title')[0].text
2366 officialDate = itemEl.findall('./pubDate')[0].text
2367
2368 configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
2369 urllib.urlencode({'uri': mediaId}))
2370 configReq = urllib2.Request(configUrl)
2371 self.report_config_download(epTitle)
2372 try:
2373 configXml = urllib2.urlopen(configReq).read()
2374 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2375 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
2376 return
2377
2378 cdoc = xml.etree.ElementTree.fromstring(configXml)
2379 turls = []
2380 for rendition in cdoc.findall('.//rendition'):
2381 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
2382 turls.append(finfo)
2383
2384 if len(turls) == 0:
2385 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
2386 continue
2387
2388 # For now, just pick the highest bitrate
2389
2390 print turls
2391
2392 if self._downloader.params.get('listformats', None):
2393 self._print_formats([i[0] for i in turls])
2394 return
2395
2396 format,video_url = turls[-1]
2397 req_format = self._downloader.params.get('format', None)
2398
2399 for f,v in turls:
2400 if f == req_format:
2401 format, video_url = f, v
2402 break
2403
2404 # Patch to download from alternative CDN, which does not
2405 # break on current RTMPDump builds
2406
2407
2408 broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/"
2409 better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/"
2410
2411 if video_url.startswith(broken_cdn):
2412 video_url = video_url.replace(broken_cdn, better_cdn)
2413
2414
2415 effTitle = showId + u'-' + epTitle
2416 info = {
2417 'id': shortMediaId,
2418 'url': video_url,
2419 'uploader': showId,
2420 'upload_date': officialDate,
2421 'title': effTitle,
2422 'ext': 'mp4',
2423 'format': format,
2424 'thumbnail': None,
2425 'description': officialTitle,
2426 'player_url': None #playerUrl
2427 }
2428
2429 results.append(info)
2430
2431 return results
2432
2433
2434 class EscapistIE(InfoExtractor):
2435 """Information extractor for The Escapist """
2436
2437 _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
2438 IE_NAME = u'escapist'
2439
2440 def report_extraction(self, showName):
2441 self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
2442
2443 def report_config_download(self, showName):
2444 self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
2445
2446 def _real_extract(self, url):
2447 mobj = re.match(self._VALID_URL, url)
2448 if mobj is None:
2449 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2450 return
2451 showName = mobj.group('showname')
2452 videoId = mobj.group('episode')
2453
2454 self.report_extraction(showName)
2455 try:
2456 webPage = urllib2.urlopen(url)
2457 webPageBytes = webPage.read()
2458 m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
2459 webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
2460 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2461 self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
2462 return
2463
2464 descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
2465 description = unescapeHTML(descMatch.group(1))
2466 imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
2467 imgUrl = unescapeHTML(imgMatch.group(1))
2468 playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
2469 playerUrl = unescapeHTML(playerUrlMatch.group(1))
2470 configUrlMatch = re.search('config=(.*)$', playerUrl)
2471 configUrl = urllib2.unquote(configUrlMatch.group(1))
2472
2473 self.report_config_download(showName)
2474 try:
2475 configJSON = urllib2.urlopen(configUrl).read()
2476 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2477 self._downloader.trouble(u'ERROR: unable to download configuration: ' + unicode(err))
2478 return
2479
2480 # Technically, it's JavaScript, not JSON
2481 configJSON = configJSON.replace("'", '"')
2482
2483 try:
2484 config = json.loads(configJSON)
2485 except (ValueError,), err:
2486 self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + unicode(err))
2487 return
2488
2489 playlist = config['playlist']
2490 videoUrl = playlist[1]['url']
2491
2492 info = {
2493 'id': videoId,
2494 'url': videoUrl,
2495 'uploader': showName,
2496 'upload_date': None,
2497 'title': showName,
2498 'ext': 'flv',
2499 'format': 'flv',
2500 'thumbnail': imgUrl,
2501 'description': description,
2502 'player_url': playerUrl,
2503 }
2504
2505 return [info]
2506
2507
2508 class CollegeHumorIE(InfoExtractor):
2509 """Information extractor for collegehumor.com"""
2510
2511 _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
2512 IE_NAME = u'collegehumor'
2513
2514 def report_webpage(self, video_id):
2515 """Report information extraction."""
2516 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2517
2518 def report_extraction(self, video_id):
2519 """Report information extraction."""
2520 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2521
2522 def _real_extract(self, url):
2523 mobj = re.match(self._VALID_URL, url)
2524 if mobj is None:
2525 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2526 return
2527 video_id = mobj.group('videoid')
2528
2529 self.report_webpage(video_id)
2530 request = urllib2.Request(url)
2531 try:
2532 webpage = urllib2.urlopen(request).read()
2533 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2534 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2535 return
2536
2537 m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
2538 if m is None:
2539 self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
2540 return
2541 internal_video_id = m.group('internalvideoid')
2542
2543 info = {
2544 'id': video_id,
2545 'internal_id': internal_video_id,
2546 }
2547
2548 self.report_extraction(video_id)
2549 xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
2550 try:
2551 metaXml = urllib2.urlopen(xmlUrl).read()
2552 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2553 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
2554 return
2555
2556 mdoc = xml.etree.ElementTree.fromstring(metaXml)
2557 try:
2558 videoNode = mdoc.findall('./video')[0]
2559 info['description'] = videoNode.findall('./description')[0].text
2560 info['title'] = videoNode.findall('./caption')[0].text
2561 info['url'] = videoNode.findall('./file')[0].text
2562 info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
2563 info['ext'] = info['url'].rpartition('.')[2]
2564 info['format'] = info['ext']
2565 except IndexError:
2566 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2567 return
2568
2569 return [info]
2570
2571
2572 class XVideosIE(InfoExtractor):
2573 """Information extractor for xvideos.com"""
2574
2575 _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
2576 IE_NAME = u'xvideos'
2577
2578 def report_webpage(self, video_id):
2579 """Report information extraction."""
2580 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2581
2582 def report_extraction(self, video_id):
2583 """Report information extraction."""
2584 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2585
2586 def _real_extract(self, url):
2587 mobj = re.match(self._VALID_URL, url)
2588 if mobj is None:
2589 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2590 return
2591 video_id = mobj.group(1).decode('utf-8')
2592
2593 self.report_webpage(video_id)
2594
2595 request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
2596 try:
2597 webpage = urllib2.urlopen(request).read()
2598 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2599 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2600 return
2601
2602 self.report_extraction(video_id)
2603
2604
2605 # Extract video URL
2606 mobj = re.search(r'flv_url=(.+?)&', webpage)
2607 if mobj is None:
2608 self._downloader.trouble(u'ERROR: unable to extract video url')
2609 return
2610 video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
2611
2612
2613 # Extract title
2614 mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
2615 if mobj is None:
2616 self._downloader.trouble(u'ERROR: unable to extract video title')
2617 return
2618 video_title = mobj.group(1).decode('utf-8')
2619
2620
2621 # Extract video thumbnail
2622 mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
2623 if mobj is None:
2624 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
2625 return
2626 video_thumbnail = mobj.group(0).decode('utf-8')
2627
2628 info = {
2629 'id': video_id,
2630 'url': video_url,
2631 'uploader': None,
2632 'upload_date': None,
2633 'title': video_title,
2634 'ext': 'flv',
2635 'format': 'flv',
2636 'thumbnail': video_thumbnail,
2637 'description': None,
2638 'player_url': None,
2639 }
2640
2641 return [info]
2642
2643
2644 class SoundcloudIE(InfoExtractor):
2645 """Information extractor for soundcloud.com
2646 To access the media, the uid of the song and a stream token
2647 must be extracted from the page source and the script must make
2648 a request to media.soundcloud.com/crossdomain.xml. Then
2649 the media can be grabbed by requesting from an url composed
2650 of the stream token and uid
2651 """
2652
2653 _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
2654 IE_NAME = u'soundcloud'
2655
2656 def __init__(self, downloader=None):
2657 InfoExtractor.__init__(self, downloader)
2658
2659 def report_webpage(self, video_id):
2660 """Report information extraction."""
2661 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2662
2663 def report_extraction(self, video_id):
2664 """Report information extraction."""
2665 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2666
2667 def _real_extract(self, url):
2668 mobj = re.match(self._VALID_URL, url)
2669 if mobj is None:
2670 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2671 return
2672
2673 # extract uploader (which is in the url)
2674 uploader = mobj.group(1).decode('utf-8')
2675 # extract simple title (uploader + slug of song title)
2676 slug_title = mobj.group(2).decode('utf-8')
2677 simple_title = uploader + u'-' + slug_title
2678
2679 self.report_webpage('%s/%s' % (uploader, slug_title))
2680
2681 request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
2682 try:
2683 webpage = urllib2.urlopen(request).read()
2684 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2685 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2686 return
2687
2688 self.report_extraction('%s/%s' % (uploader, slug_title))
2689
2690 # extract uid and stream token that soundcloud hands out for access
2691 mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
2692 if mobj:
2693 video_id = mobj.group(1)
2694 stream_token = mobj.group(2)
2695
2696 # extract unsimplified title
2697 mobj = re.search('"title":"(.*?)",', webpage)
2698 if mobj:
2699 title = mobj.group(1).decode('utf-8')
2700 else:
2701 title = simple_title
2702
2703 # construct media url (with uid/token)
2704 mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
2705 mediaURL = mediaURL % (video_id, stream_token)
2706
2707 # description
2708 description = u'No description available'
2709 mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
2710 if mobj:
2711 description = mobj.group(1)
2712
2713 # upload date
2714 upload_date = None
2715 mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
2716 if mobj:
2717 try:
2718 upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
2719 except Exception, e:
2720 self._downloader.to_stderr(compat_str(e))
2721
2722 # for soundcloud, a request to a cross domain is required for cookies
2723 request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
2724
2725 return [{
2726 'id': video_id.decode('utf-8'),
2727 'url': mediaURL,
2728 'uploader': uploader.decode('utf-8'),
2729 'upload_date': upload_date,
2730 'title': title,
2731 'ext': u'mp3',
2732 'format': u'NA',
2733 'player_url': None,
2734 'description': description.decode('utf-8')
2735 }]
2736
2737
2738 class InfoQIE(InfoExtractor):
2739 """Information extractor for infoq.com"""
2740
2741 _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
2742 IE_NAME = u'infoq'
2743
2744 def report_webpage(self, video_id):
2745 """Report information extraction."""
2746 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2747
2748 def report_extraction(self, video_id):
2749 """Report information extraction."""
2750 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2751
2752 def _real_extract(self, url):
2753 mobj = re.match(self._VALID_URL, url)
2754 if mobj is None:
2755 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2756 return
2757
2758 self.report_webpage(url)
2759
2760 request = urllib2.Request(url)
2761 try:
2762 webpage = urllib2.urlopen(request).read()
2763 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2764 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2765 return
2766
2767 self.report_extraction(url)
2768
2769
2770 # Extract video URL
2771 mobj = re.search(r"jsclassref='([^']*)'", webpage)
2772 if mobj is None:
2773 self._downloader.trouble(u'ERROR: unable to extract video url')
2774 return
2775 video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
2776
2777
2778 # Extract title
2779 mobj = re.search(r'contentTitle = "(.*?)";', webpage)
2780 if mobj is None:
2781 self._downloader.trouble(u'ERROR: unable to extract video title')
2782 return
2783 video_title = mobj.group(1).decode('utf-8')
2784
2785 # Extract description
2786 video_description = u'No description available.'
2787 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
2788 if mobj is not None:
2789 video_description = mobj.group(1).decode('utf-8')
2790
2791 video_filename = video_url.split('/')[-1]
2792 video_id, extension = video_filename.split('.')
2793
2794 info = {
2795 'id': video_id,
2796 'url': video_url,
2797 'uploader': None,
2798 'upload_date': None,
2799 'title': video_title,
2800 'ext': extension,
2801 'format': extension, # Extension is always(?) mp4, but seems to be flv
2802 'thumbnail': None,
2803 'description': video_description,
2804 'player_url': None,
2805 }
2806
2807 return [info]
2808
2809 class MixcloudIE(InfoExtractor):
2810 """Information extractor for www.mixcloud.com"""
2811 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
2812 IE_NAME = u'mixcloud'
2813
2814 def __init__(self, downloader=None):
2815 InfoExtractor.__init__(self, downloader)
2816
2817 def report_download_json(self, file_id):
2818 """Report JSON download."""
2819 self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
2820
2821 def report_extraction(self, file_id):
2822 """Report information extraction."""
2823 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2824
2825 def get_urls(self, jsonData, fmt, bitrate='best'):
2826 """Get urls from 'audio_formats' section in json"""
2827 file_url = None
2828 try:
2829 bitrate_list = jsonData[fmt]
2830 if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
2831 bitrate = max(bitrate_list) # select highest
2832
2833 url_list = jsonData[fmt][bitrate]
2834 except TypeError: # we have no bitrate info.
2835 url_list = jsonData[fmt]
2836 return url_list
2837
2838 def check_urls(self, url_list):
2839 """Returns 1st active url from list"""
2840 for url in url_list:
2841 try:
2842 urllib2.urlopen(url)
2843 return url
2844 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2845 url = None
2846
2847 return None
2848
2849 def _print_formats(self, formats):
2850 print('Available formats:')
2851 for fmt in formats.keys():
2852 for b in formats[fmt]:
2853 try:
2854 ext = formats[fmt][b][0]
2855 print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
2856 except TypeError: # we have no bitrate info
2857 ext = formats[fmt][0]
2858 print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
2859 break
2860
2861 def _real_extract(self, url):
2862 mobj = re.match(self._VALID_URL, url)
2863 if mobj is None:
2864 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2865 return
2866 # extract uploader & filename from url
2867 uploader = mobj.group(1).decode('utf-8')
2868 file_id = uploader + "-" + mobj.group(2).decode('utf-8')
2869
2870 # construct API request
2871 file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
2872 # retrieve .json file with links to files
2873 request = urllib2.Request(file_url)
2874 try:
2875 self.report_download_json(file_url)
2876 jsonData = urllib2.urlopen(request).read()
2877 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2878 self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
2879 return
2880
2881 # parse JSON
2882 json_data = json.loads(jsonData)
2883 player_url = json_data['player_swf_url']
2884 formats = dict(json_data['audio_formats'])
2885
2886 req_format = self._downloader.params.get('format', None)
2887 bitrate = None
2888
2889 if self._downloader.params.get('listformats', None):
2890 self._print_formats(formats)
2891 return
2892
2893 if req_format is None or req_format == 'best':
2894 for format_param in formats.keys():
2895 url_list = self.get_urls(formats, format_param)
2896 # check urls
2897 file_url = self.check_urls(url_list)
2898 if file_url is not None:
2899 break # got it!
2900 else:
2901 if req_format not in formats.keys():
2902 self._downloader.trouble(u'ERROR: format is not available')
2903 return
2904
2905 url_list = self.get_urls(formats, req_format)
2906 file_url = self.check_urls(url_list)
2907 format_param = req_format
2908
2909 return [{
2910 'id': file_id.decode('utf-8'),
2911 'url': file_url.decode('utf-8'),
2912 'uploader': uploader.decode('utf-8'),
2913 'upload_date': u'NA',
2914 'title': json_data['name'],
2915 'ext': file_url.split('.')[-1].decode('utf-8'),
2916 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
2917 'thumbnail': json_data['thumbnail_url'],
2918 'description': json_data['description'],
2919 'player_url': player_url.decode('utf-8'),
2920 }]
2921
2922 class StanfordOpenClassroomIE(InfoExtractor):
2923 """Information extractor for Stanford's Open ClassRoom"""
2924
2925 _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
2926 IE_NAME = u'stanfordoc'
2927
2928 def report_download_webpage(self, objid):
2929 """Report information extraction."""
2930 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
2931
2932 def report_extraction(self, video_id):
2933 """Report information extraction."""
2934 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2935
2936 def _real_extract(self, url):
2937 mobj = re.match(self._VALID_URL, url)
2938 if mobj is None:
2939 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2940 return
2941
2942 if mobj.group('course') and mobj.group('video'): # A specific video
2943 course = mobj.group('course')
2944 video = mobj.group('video')
2945 info = {
2946 'id': course + '_' + video,
2947 }
2948
2949 self.report_extraction(info['id'])
2950 baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
2951 xmlUrl = baseUrl + video + '.xml'
2952 try:
2953 metaXml = urllib2.urlopen(xmlUrl).read()
2954 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2955 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err))
2956 return
2957 mdoc = xml.etree.ElementTree.fromstring(metaXml)
2958 try:
2959 info['title'] = mdoc.findall('./title')[0].text
2960 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
2961 except IndexError:
2962 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2963 return
2964 info['ext'] = info['url'].rpartition('.')[2]
2965 info['format'] = info['ext']
2966 return [info]
2967 elif mobj.group('course'): # A course page
2968 course = mobj.group('course')
2969 info = {
2970 'id': course,
2971 'type': 'playlist',
2972 }
2973
2974 self.report_download_webpage(info['id'])
2975 try:
2976 coursepage = urllib2.urlopen(url).read()
2977 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2978 self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
2979 return
2980
2981 m = re.search('<h1>([^<]+)</h1>', coursepage)
2982 if m:
2983 info['title'] = unescapeHTML(m.group(1))
2984 else:
2985 info['title'] = info['id']
2986
2987 m = re.search('<description>([^<]+)</description>', coursepage)
2988 if m:
2989 info['description'] = unescapeHTML(m.group(1))
2990
2991 links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
2992 info['list'] = [
2993 {
2994 'type': 'reference',
2995 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
2996 }
2997 for vpage in links]
2998 results = []
2999 for entry in info['list']:
3000 assert entry['type'] == 'reference'
3001 results += self.extract(entry['url'])
3002 return results
3003
3004 else: # Root page
3005 info = {
3006 'id': 'Stanford OpenClassroom',
3007 'type': 'playlist',
3008 }
3009
3010 self.report_download_webpage(info['id'])
3011 rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
3012 try:
3013 rootpage = urllib2.urlopen(rootURL).read()
3014 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3015 self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
3016 return
3017
3018 info['title'] = info['id']
3019
3020 links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
3021 info['list'] = [
3022 {
3023 'type': 'reference',
3024 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
3025 }
3026 for cpage in links]
3027
3028 results = []
3029 for entry in info['list']:
3030 assert entry['type'] == 'reference'
3031 results += self.extract(entry['url'])
3032 return results
3033
3034 class MTVIE(InfoExtractor):
3035 """Information extractor for MTV.com"""
3036
3037 _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
3038 IE_NAME = u'mtv'
3039
3040 def report_webpage(self, video_id):
3041 """Report information extraction."""
3042 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3043
3044 def report_extraction(self, video_id):
3045 """Report information extraction."""
3046 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3047
3048 def _real_extract(self, url):
3049 mobj = re.match(self._VALID_URL, url)
3050 if mobj is None:
3051 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3052 return
3053 if not mobj.group('proto'):
3054 url = 'http://' + url
3055 video_id = mobj.group('videoid')
3056 self.report_webpage(video_id)
3057
3058 request = urllib2.Request(url)
3059 try:
3060 webpage = urllib2.urlopen(request).read()
3061 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3062 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
3063 return
3064
3065 mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
3066 if mobj is None:
3067 self._downloader.trouble(u'ERROR: unable to extract song name')
3068 return
3069 song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3070 mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
3071 if mobj is None:
3072 self._downloader.trouble(u'ERROR: unable to extract performer')
3073 return
3074 performer = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3075 video_title = performer + ' - ' + song_name
3076
3077 mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
3078 if mobj is None:
3079 self._downloader.trouble(u'ERROR: unable to mtvn_uri')
3080 return
3081 mtvn_uri = mobj.group(1)
3082
3083 mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
3084 if mobj is None:
3085 self._downloader.trouble(u'ERROR: unable to extract content id')
3086 return
3087 content_id = mobj.group(1)
3088
3089 videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
3090 self.report_extraction(video_id)
3091 request = urllib2.Request(videogen_url)
3092 try:
3093 metadataXml = urllib2.urlopen(request).read()
3094 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3095 self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
3096 return
3097
3098 mdoc = xml.etree.ElementTree.fromstring(metadataXml)
3099 renditions = mdoc.findall('.//rendition')
3100
3101 # For now, always pick the highest quality.
3102 rendition = renditions[-1]
3103
3104 try:
3105 _,_,ext = rendition.attrib['type'].partition('/')
3106 format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
3107 video_url = rendition.find('./src').text
3108 except KeyError:
3109 self._downloader.trouble('Invalid rendition field.')
3110 return
3111
3112 info = {
3113 'id': video_id,
3114 'url': video_url,
3115 'uploader': performer,
3116 'title': video_title,
3117 'ext': ext,
3118 'format': format,
3119 }
3120
3121 return [info]
3122
3123
3124 class YoukuIE(InfoExtractor):
3125
3126 _VALID_URL = r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
3127 IE_NAME = u'Youku'
3128
3129 def __init__(self, downloader=None):
3130 InfoExtractor.__init__(self, downloader)
3131
3132 def report_download_webpage(self, file_id):
3133 """Report webpage download."""
3134 self._downloader.to_screen(u'[Youku] %s: Downloading webpage' % file_id)
3135
3136 def report_extraction(self, file_id):
3137 """Report information extraction."""
3138 self._downloader.to_screen(u'[Youku] %s: Extracting information' % file_id)
3139
3140 def _gen_sid(self):
3141 nowTime = int(time.time() * 1000)
3142 random1 = random.randint(1000,1998)
3143 random2 = random.randint(1000,9999)
3144
3145 return "%d%d%d" %(nowTime,random1,random2)
3146
3147 def _get_file_ID_mix_string(self, seed):
3148 mixed = []
3149 source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
3150 seed = float(seed)
3151 for i in range(len(source)):
3152 seed = (seed * 211 + 30031 ) % 65536
3153 index = math.floor(seed / 65536 * len(source) )
3154 mixed.append(source[int(index)])
3155 source.remove(source[int(index)])
3156 #return ''.join(mixed)
3157 return mixed
3158
3159 def _get_file_id(self, fileId, seed):
3160 mixed = self._get_file_ID_mix_string(seed)
3161 ids = fileId.split('*')
3162 realId = []
3163 for ch in ids:
3164 if ch:
3165 realId.append(mixed[int(ch)])
3166 return ''.join(realId)
3167
3168 def _real_extract(self, url):
3169 mobj = re.match(self._VALID_URL, url)
3170 if mobj is None:
3171 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3172 return
3173 video_id = mobj.group('ID')
3174
3175 info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
3176
3177 request = urllib2.Request(info_url, None, std_headers)
3178 try:
3179 self.report_download_webpage(video_id)
3180 jsondata = urllib2.urlopen(request).read()
3181 except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
3182 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
3183 return
3184
3185 self.report_extraction(video_id)
3186 try:
3187 config = json.loads(jsondata)
3188
3189 video_title = config['data'][0]['title']
3190 seed = config['data'][0]['seed']
3191
3192 format = self._downloader.params.get('format', None)
3193 supported_format = config['data'][0]['streamfileids'].keys()
3194
3195 if format is None or format == 'best':
3196 if 'hd2' in supported_format:
3197 format = 'hd2'
3198 else:
3199 format = 'flv'
3200 ext = u'flv'
3201 elif format == 'worst':
3202 format = 'mp4'
3203 ext = u'mp4'
3204 else:
3205 format = 'flv'
3206 ext = u'flv'
3207
3208
3209 fileid = config['data'][0]['streamfileids'][format]
3210 seg_number = len(config['data'][0]['segs'][format])
3211
3212 keys=[]
3213 for i in xrange(seg_number):
3214 keys.append(config['data'][0]['segs'][format][i]['k'])
3215
3216 #TODO check error
3217 #youku only could be viewed from mainland china
3218 except:
3219 self._downloader.trouble(u'ERROR: unable to extract info section')
3220 return
3221
3222 files_info=[]
3223 sid = self._gen_sid()
3224 fileid = self._get_file_id(fileid, seed)
3225
3226 #column 8,9 of fileid represent the segment number
3227 #fileid[7:9] should be changed
3228 for index, key in enumerate(keys):
3229
3230 temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
3231 download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
3232
3233 info = {
3234 'id': '%s_part%02d' % (video_id, index),
3235 'url': download_url,
3236 'uploader': None,
3237 'title': video_title,
3238 'ext': ext,
3239 'format': u'NA'
3240 }
3241 files_info.append(info)
3242
3243 return files_info
3244
3245
3246 class XNXXIE(InfoExtractor):
3247 """Information extractor for xnxx.com"""
3248
3249 _VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)'
3250 IE_NAME = u'xnxx'
3251 VIDEO_URL_RE = r'flv_url=(.*?)&amp;'
3252 VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
3253 VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&amp;'
3254
3255 def report_webpage(self, video_id):
3256 """Report information extraction"""
3257 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3258
3259 def report_extraction(self, video_id):
3260 """Report information extraction"""
3261 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3262
3263 def _real_extract(self, url):
3264 mobj = re.match(self._VALID_URL, url)
3265 if mobj is None:
3266 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3267 return
3268 video_id = mobj.group(1).decode('utf-8')
3269
3270 self.report_webpage(video_id)
3271
3272 # Get webpage content
3273 try:
3274 webpage = urllib2.urlopen(url).read()
3275 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3276 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
3277 return
3278
3279 result = re.search(self.VIDEO_URL_RE, webpage)
3280 if result is None:
3281 self._downloader.trouble(u'ERROR: unable to extract video url')
3282 return
3283 video_url = urllib.unquote(result.group(1).decode('utf-8'))
3284
3285 result = re.search(self.VIDEO_TITLE_RE, webpage)
3286 if result is None:
3287 self._downloader.trouble(u'ERROR: unable to extract video title')
3288 return
3289 video_title = result.group(1).decode('utf-8')
3290
3291 result = re.search(self.VIDEO_THUMB_RE, webpage)
3292 if result is None:
3293 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
3294 return
3295 video_thumbnail = result.group(1).decode('utf-8')
3296
3297 info = {'id': video_id,
3298 'url': video_url,
3299 'uploader': None,
3300 'upload_date': None,
3301 'title': video_title,
3302 'ext': 'flv',
3303 'format': 'flv',
3304 'thumbnail': video_thumbnail,
3305 'description': None,
3306 'player_url': None}
3307
3308 return [info]
3309
3310
3311 class GooglePlusIE(InfoExtractor):
3312 """Information extractor for plus.google.com."""
3313
3314 _VALID_URL = r'(?:https://)?plus\.google\.com/(?:\w+/)*?(\d+)/posts/(\w+)'
3315 IE_NAME = u'plus.google'
3316
3317 def __init__(self, downloader=None):
3318 InfoExtractor.__init__(self, downloader)
3319
3320 def report_extract_entry(self, url):
3321 """Report downloading extry"""
3322 self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url.decode('utf-8'))
3323
3324 def report_date(self, upload_date):
3325 """Report downloading extry"""
3326 self._downloader.to_screen(u'[plus.google] Entry date: %s' % upload_date)
3327
3328 def report_uploader(self, uploader):
3329 """Report downloading extry"""
3330 self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader.decode('utf-8'))
3331
3332 def report_title(self, video_title):
3333 """Report downloading extry"""
3334 self._downloader.to_screen(u'[plus.google] Title: %s' % video_title.decode('utf-8'))
3335
3336 def report_extract_vid_page(self, video_page):
3337 """Report information extraction."""
3338 self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page.decode('utf-8'))
3339
3340 def _real_extract(self, url):
3341 # Extract id from URL
3342 mobj = re.match(self._VALID_URL, url)
3343 if mobj is None:
3344 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
3345 return
3346
3347 post_url = mobj.group(0)
3348 video_id = mobj.group(2)
3349
3350 video_extension = 'flv'
3351
3352 # Step 1, Retrieve post webpage to extract further information
3353 self.report_extract_entry(post_url)
3354 request = urllib2.Request(post_url)
3355 try:
3356 webpage = urllib2.urlopen(request).read()
3357 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3358 self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
3359 return
3360
3361 # Extract update date
3362 upload_date = u'NA'
3363 pattern = 'title="Timestamp">(.*?)</a>'
3364 mobj = re.search(pattern, webpage)
3365 if mobj:
3366 upload_date = mobj.group(1)
3367 # Convert timestring to a format suitable for filename
3368 upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
3369 upload_date = upload_date.strftime('%Y%m%d')
3370 self.report_date(upload_date)
3371
3372 # Extract uploader
3373 uploader = u'NA'
3374 pattern = r'rel\="author".*?>(.*?)</a>'
3375 mobj = re.search(pattern, webpage)
3376 if mobj:
3377 uploader = mobj.group(1)
3378 self.report_uploader(uploader)
3379
3380 # Extract title
3381 # Get the first line for title
3382 video_title = u'NA'
3383 pattern = r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]'
3384 mobj = re.search(pattern, webpage)
3385 if mobj:
3386 video_title = mobj.group(1)
3387 self.report_title(video_title)
3388
3389 # Step 2, Stimulate clicking the image box to launch video
3390 pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]'
3391 mobj = re.search(pattern, webpage)
3392 if mobj is None:
3393 self._downloader.trouble(u'ERROR: unable to extract video page URL')
3394
3395 video_page = mobj.group(1)
3396 request = urllib2.Request(video_page)
3397 try:
3398 webpage = urllib2.urlopen(request).read()
3399 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3400 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
3401 return
3402 self.report_extract_vid_page(video_page)
3403
3404
3405 # Extract video links on video page
3406 """Extract video links of all sizes"""
3407 pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
3408 mobj = re.findall(pattern, webpage)
3409 if len(mobj) == 0:
3410 self._downloader.trouble(u'ERROR: unable to extract video links')
3411
3412 # Sort in resolution
3413 links = sorted(mobj)
3414
3415 # Choose the lowest of the sort, i.e. highest resolution
3416 video_url = links[-1]
3417 # Only get the url. The resolution part in the tuple has no use anymore
3418 video_url = video_url[-1]
3419 # Treat escaped \u0026 style hex
3420 video_url = unicode(video_url, "unicode_escape")
3421
3422
3423 return [{
3424 'id': video_id.decode('utf-8'),
3425 'url': video_url,
3426 'uploader': uploader.decode('utf-8'),
3427 'upload_date': upload_date.decode('utf-8'),
3428 'title': video_title.decode('utf-8'),
3429 'ext': video_extension.decode('utf-8'),
3430 'format': u'NA',
3431 'player_url': None,
3432 }]
fork of https://github.com/yt-dlp/yt-dlp/