]> jfr.im git - yt-dlp.git/blob - youtube_dl/__init__.py
Bump version number to a numeric-only one to appease py2exe
[yt-dlp.git] / youtube_dl / __init__.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import with_statement
5
6 __authors__ = (
7 'Ricardo Garcia Gonzalez',
8 'Danny Colligan',
9 'Benjamin Johnson',
10 'Vasyl\' Vavrychuk',
11 'Witold Baryluk',
12 'Paweł Paprota',
13 'Gergely Imreh',
14 'Rogério Brito',
15 'Philipp Hagemeister',
16 'Sören Schulze',
17 'Kevin Ngo',
18 'Ori Avtalion',
19 'shizeeg',
20 'Filippo Valsorda',
21 )
22
23 __license__ = 'Public Domain'
24 __version__ = '2012.11.28'
25
26 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
27 UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION'
28 UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe'
29
30
31 import cookielib
32 import getpass
33 import optparse
34 import os
35 import re
36 import shlex
37 import socket
38 import subprocess
39 import sys
40 import urllib2
41 import warnings
42
43 from utils import *
44 from FileDownloader import *
45 from InfoExtractors import *
46 from PostProcessor import *
47
48 def updateSelf(downloader, filename):
49 ''' Update the program file with the latest version from the repository '''
50 # Note: downloader only used for options
51
52 if not os.access(filename, os.W_OK):
53 sys.exit('ERROR: no write permissions on %s' % filename)
54
55 downloader.to_screen(u'Updating to latest version...')
56
57 urlv = urllib2.urlopen(UPDATE_URL_VERSION)
58 newversion = urlv.read().strip()
59 if newversion == __version__:
60 downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
61 return
62 urlv.close()
63
64 if hasattr(sys, "frozen"): #py2exe
65 exe = os.path.abspath(filename)
66 directory = os.path.dirname(exe)
67 if not os.access(directory, os.W_OK):
68 sys.exit('ERROR: no write permissions on %s' % directory)
69
70 try:
71 urlh = urllib2.urlopen(UPDATE_URL_EXE)
72 newcontent = urlh.read()
73 urlh.close()
74 with open(exe + '.new', 'wb') as outf:
75 outf.write(newcontent)
76 except (IOError, OSError), err:
77 sys.exit('ERROR: unable to download latest version')
78
79 try:
80 bat = os.path.join(directory, 'youtube-dl-updater.bat')
81 b = open(bat, 'w')
82 b.write("""
83 echo Updating youtube-dl...
84 ping 127.0.0.1 -n 5 -w 1000 > NUL
85 move /Y "%s.new" "%s"
86 del "%s"
87 \n""" %(exe, exe, bat))
88 b.close()
89
90 os.startfile(bat)
91 except (IOError, OSError), err:
92 sys.exit('ERROR: unable to overwrite current version')
93
94 else:
95 try:
96 urlh = urllib2.urlopen(UPDATE_URL)
97 newcontent = urlh.read()
98 urlh.close()
99 except (IOError, OSError), err:
100 sys.exit('ERROR: unable to download latest version')
101
102 try:
103 with open(filename, 'wb') as outf:
104 outf.write(newcontent)
105 except (IOError, OSError), err:
106 sys.exit('ERROR: unable to overwrite current version')
107
108 downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
109
110 def parseOpts():
111 def _readOptions(filename_bytes):
112 try:
113 optionf = open(filename_bytes)
114 except IOError:
115 return [] # silently skip if file is not present
116 try:
117 res = []
118 for l in optionf:
119 res += shlex.split(l, comments=True)
120 finally:
121 optionf.close()
122 return res
123
124 def _format_option_string(option):
125 ''' ('-o', '--option') -> -o, --format METAVAR'''
126
127 opts = []
128
129 if option._short_opts: opts.append(option._short_opts[0])
130 if option._long_opts: opts.append(option._long_opts[0])
131 if len(opts) > 1: opts.insert(1, ', ')
132
133 if option.takes_value(): opts.append(' %s' % option.metavar)
134
135 return "".join(opts)
136
137 def _find_term_columns():
138 columns = os.environ.get('COLUMNS', None)
139 if columns:
140 return int(columns)
141
142 try:
143 sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
144 out,err = sp.communicate()
145 return int(out.split()[1])
146 except:
147 pass
148 return None
149
150 max_width = 80
151 max_help_position = 80
152
153 # No need to wrap help messages if we're on a wide console
154 columns = _find_term_columns()
155 if columns: max_width = columns
156
157 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
158 fmt.format_option_strings = _format_option_string
159
160 kw = {
161 'version' : __version__,
162 'formatter' : fmt,
163 'usage' : '%prog [options] url [url...]',
164 'conflict_handler' : 'resolve',
165 }
166
167 parser = optparse.OptionParser(**kw)
168
169 # option groups
170 general = optparse.OptionGroup(parser, 'General Options')
171 selection = optparse.OptionGroup(parser, 'Video Selection')
172 authentication = optparse.OptionGroup(parser, 'Authentication Options')
173 video_format = optparse.OptionGroup(parser, 'Video Format Options')
174 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
175 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
176 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
177
178 general.add_option('-h', '--help',
179 action='help', help='print this help text and exit')
180 general.add_option('-v', '--version',
181 action='version', help='print program version and exit')
182 general.add_option('-U', '--update',
183 action='store_true', dest='update_self', help='update this program to latest version')
184 general.add_option('-i', '--ignore-errors',
185 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
186 general.add_option('-r', '--rate-limit',
187 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
188 general.add_option('-R', '--retries',
189 dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
190 general.add_option('--dump-user-agent',
191 action='store_true', dest='dump_user_agent',
192 help='display the current browser identification', default=False)
193 general.add_option('--user-agent',
194 dest='user_agent', help='specify a custom user agent', metavar='UA')
195 general.add_option('--list-extractors',
196 action='store_true', dest='list_extractors',
197 help='List all supported extractors and the URLs they would handle', default=False)
198
199 selection.add_option('--playlist-start',
200 dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
201 selection.add_option('--playlist-end',
202 dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
203 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
204 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
205 selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
206
207 authentication.add_option('-u', '--username',
208 dest='username', metavar='USERNAME', help='account username')
209 authentication.add_option('-p', '--password',
210 dest='password', metavar='PASSWORD', help='account password')
211 authentication.add_option('-n', '--netrc',
212 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
213
214
215 video_format.add_option('-f', '--format',
216 action='store', dest='format', metavar='FORMAT', help='video format code')
217 video_format.add_option('--all-formats',
218 action='store_const', dest='format', help='download all available video formats', const='all')
219 video_format.add_option('--prefer-free-formats',
220 action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
221 video_format.add_option('--max-quality',
222 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
223 video_format.add_option('-F', '--list-formats',
224 action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
225 video_format.add_option('--write-srt',
226 action='store_true', dest='writesubtitles',
227 help='write video closed captions to a .srt file (currently youtube only)', default=False)
228 video_format.add_option('--srt-lang',
229 action='store', dest='subtitleslang', metavar='LANG',
230 help='language of the closed captions to download (optional) use IETF language tags like \'en\'')
231
232
233 verbosity.add_option('-q', '--quiet',
234 action='store_true', dest='quiet', help='activates quiet mode', default=False)
235 verbosity.add_option('-s', '--simulate',
236 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
237 verbosity.add_option('--skip-download',
238 action='store_true', dest='skip_download', help='do not download the video', default=False)
239 verbosity.add_option('-g', '--get-url',
240 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
241 verbosity.add_option('-e', '--get-title',
242 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
243 verbosity.add_option('--get-thumbnail',
244 action='store_true', dest='getthumbnail',
245 help='simulate, quiet but print thumbnail URL', default=False)
246 verbosity.add_option('--get-description',
247 action='store_true', dest='getdescription',
248 help='simulate, quiet but print video description', default=False)
249 verbosity.add_option('--get-filename',
250 action='store_true', dest='getfilename',
251 help='simulate, quiet but print output filename', default=False)
252 verbosity.add_option('--get-format',
253 action='store_true', dest='getformat',
254 help='simulate, quiet but print output format', default=False)
255 verbosity.add_option('--no-progress',
256 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
257 verbosity.add_option('--console-title',
258 action='store_true', dest='consoletitle',
259 help='display progress in console titlebar', default=False)
260 verbosity.add_option('-v', '--verbose',
261 action='store_true', dest='verbose', help='print various debugging information', default=False)
262
263
264 filesystem.add_option('-t', '--title',
265 action='store_true', dest='usetitle', help='use title in file name', default=False)
266 filesystem.add_option('--id',
267 action='store_true', dest='useid', help='use video ID in file name', default=False)
268 filesystem.add_option('-l', '--literal',
269 action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
270 filesystem.add_option('-A', '--auto-number',
271 action='store_true', dest='autonumber',
272 help='number downloaded files starting from 00000', default=False)
273 filesystem.add_option('-o', '--output',
274 dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
275 filesystem.add_option('--restrict-filenames',
276 action='store_true', dest='restrictfilenames',
277 help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
278 filesystem.add_option('-a', '--batch-file',
279 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
280 filesystem.add_option('-w', '--no-overwrites',
281 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
282 filesystem.add_option('-c', '--continue',
283 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
284 filesystem.add_option('--no-continue',
285 action='store_false', dest='continue_dl',
286 help='do not resume partially downloaded files (restart from beginning)')
287 filesystem.add_option('--cookies',
288 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
289 filesystem.add_option('--no-part',
290 action='store_true', dest='nopart', help='do not use .part files', default=False)
291 filesystem.add_option('--no-mtime',
292 action='store_false', dest='updatetime',
293 help='do not use the Last-modified header to set the file modification time', default=True)
294 filesystem.add_option('--write-description',
295 action='store_true', dest='writedescription',
296 help='write video description to a .description file', default=False)
297 filesystem.add_option('--write-info-json',
298 action='store_true', dest='writeinfojson',
299 help='write video metadata to a .info.json file', default=False)
300
301
302 postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
303 help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
304 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
305 help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default')
306 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
307 help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
308 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
309 help='keeps the video file on disk after the post-processing; the video is erased by default')
310
311
312 parser.add_option_group(general)
313 parser.add_option_group(selection)
314 parser.add_option_group(filesystem)
315 parser.add_option_group(verbosity)
316 parser.add_option_group(video_format)
317 parser.add_option_group(authentication)
318 parser.add_option_group(postproc)
319
320 xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
321 if xdg_config_home:
322 userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
323 else:
324 userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
325 argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
326 opts, args = parser.parse_args(argv)
327
328 return parser, opts, args
329
330 def gen_extractors():
331 """ Return a list of an instance of every supported extractor.
332 The order does matter; the first extractor matched is the one handling the URL.
333 """
334 return [
335 YoutubePlaylistIE(),
336 YoutubeChannelIE(),
337 YoutubeUserIE(),
338 YoutubeSearchIE(),
339 YoutubeIE(),
340 MetacafeIE(),
341 DailymotionIE(),
342 GoogleIE(),
343 GoogleSearchIE(),
344 PhotobucketIE(),
345 YahooIE(),
346 YahooSearchIE(),
347 DepositFilesIE(),
348 FacebookIE(),
349 BlipTVUserIE(),
350 BlipTVIE(),
351 VimeoIE(),
352 MyVideoIE(),
353 ComedyCentralIE(),
354 EscapistIE(),
355 CollegeHumorIE(),
356 XVideosIE(),
357 SoundcloudIE(),
358 InfoQIE(),
359 MixcloudIE(),
360 StanfordOpenClassroomIE(),
361 MTVIE(),
362 YoukuIE(),
363 XNXXIE(),
364 GooglePlusIE(),
365
366 GenericIE()
367 ]
368
369 def _real_main():
370 parser, opts, args = parseOpts()
371
372 # Open appropriate CookieJar
373 if opts.cookiefile is None:
374 jar = cookielib.CookieJar()
375 else:
376 try:
377 jar = cookielib.MozillaCookieJar(opts.cookiefile)
378 if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
379 jar.load()
380 except (IOError, OSError), err:
381 sys.exit(u'ERROR: unable to open cookie file')
382 # Set user agent
383 if opts.user_agent is not None:
384 std_headers['User-Agent'] = opts.user_agent
385
386 # Dump user agent
387 if opts.dump_user_agent:
388 print std_headers['User-Agent']
389 sys.exit(0)
390
391 # Batch file verification
392 batchurls = []
393 if opts.batchfile is not None:
394 try:
395 if opts.batchfile == '-':
396 batchfd = sys.stdin
397 else:
398 batchfd = open(opts.batchfile, 'r')
399 batchurls = batchfd.readlines()
400 batchurls = [x.strip() for x in batchurls]
401 batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
402 except IOError:
403 sys.exit(u'ERROR: batch file could not be read')
404 all_urls = batchurls + args
405 all_urls = map(lambda url: url.strip(), all_urls)
406
407 # General configuration
408 cookie_processor = urllib2.HTTPCookieProcessor(jar)
409 proxy_handler = urllib2.ProxyHandler()
410 opener = urllib2.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
411 urllib2.install_opener(opener)
412 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
413
414 extractors = gen_extractors()
415
416 if opts.list_extractors:
417 for ie in extractors:
418 print(ie.IE_NAME)
419 matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
420 all_urls = filter(lambda url: url not in matchedUrls, all_urls)
421 for mu in matchedUrls:
422 print(u' ' + mu)
423 sys.exit(0)
424
425 # Conflicting, missing and erroneous options
426 if opts.usenetrc and (opts.username is not None or opts.password is not None):
427 parser.error(u'using .netrc conflicts with giving username/password')
428 if opts.password is not None and opts.username is None:
429 parser.error(u'account username missing')
430 if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
431 parser.error(u'using output template conflicts with using title, video ID or auto number')
432 if opts.usetitle and opts.useid:
433 parser.error(u'using title conflicts with using video ID')
434 if opts.username is not None and opts.password is None:
435 opts.password = getpass.getpass(u'Type account password and press return:')
436 if opts.ratelimit is not None:
437 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
438 if numeric_limit is None:
439 parser.error(u'invalid rate limit specified')
440 opts.ratelimit = numeric_limit
441 if opts.retries is not None:
442 try:
443 opts.retries = long(opts.retries)
444 except (TypeError, ValueError), err:
445 parser.error(u'invalid retry count specified')
446 try:
447 opts.playliststart = int(opts.playliststart)
448 if opts.playliststart <= 0:
449 raise ValueError(u'Playlist start must be positive')
450 except (TypeError, ValueError), err:
451 parser.error(u'invalid playlist start number specified')
452 try:
453 opts.playlistend = int(opts.playlistend)
454 if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
455 raise ValueError(u'Playlist end must be greater than playlist start')
456 except (TypeError, ValueError), err:
457 parser.error(u'invalid playlist end number specified')
458 if opts.extractaudio:
459 if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
460 parser.error(u'invalid audio format specified')
461 if opts.audioquality:
462 opts.audioquality = opts.audioquality.strip('k').strip('K')
463 if not opts.audioquality.isdigit():
464 parser.error(u'invalid audio quality specified')
465
466 # File downloader
467 fd = FileDownloader({
468 'usenetrc': opts.usenetrc,
469 'username': opts.username,
470 'password': opts.password,
471 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
472 'forceurl': opts.geturl,
473 'forcetitle': opts.gettitle,
474 'forcethumbnail': opts.getthumbnail,
475 'forcedescription': opts.getdescription,
476 'forcefilename': opts.getfilename,
477 'forceformat': opts.getformat,
478 'simulate': opts.simulate,
479 'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
480 'format': opts.format,
481 'format_limit': opts.format_limit,
482 'listformats': opts.listformats,
483 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
484 or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
485 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
486 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
487 or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
488 or (opts.useid and u'%(id)s.%(ext)s')
489 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
490 or u'%(id)s.%(ext)s'),
491 'restrictfilenames': opts.restrictfilenames,
492 'ignoreerrors': opts.ignoreerrors,
493 'ratelimit': opts.ratelimit,
494 'nooverwrites': opts.nooverwrites,
495 'retries': opts.retries,
496 'continuedl': opts.continue_dl,
497 'noprogress': opts.noprogress,
498 'playliststart': opts.playliststart,
499 'playlistend': opts.playlistend,
500 'logtostderr': opts.outtmpl == '-',
501 'consoletitle': opts.consoletitle,
502 'nopart': opts.nopart,
503 'updatetime': opts.updatetime,
504 'writedescription': opts.writedescription,
505 'writeinfojson': opts.writeinfojson,
506 'writesubtitles': opts.writesubtitles,
507 'subtitleslang': opts.subtitleslang,
508 'matchtitle': opts.matchtitle,
509 'rejecttitle': opts.rejecttitle,
510 'max_downloads': opts.max_downloads,
511 'prefer_free_formats': opts.prefer_free_formats,
512 'verbose': opts.verbose,
513 })
514
515 if opts.verbose:
516 fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies))
517
518 for extractor in extractors:
519 fd.add_info_extractor(extractor)
520
521 # PostProcessors
522 if opts.extractaudio:
523 fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
524
525 # Update version
526 if opts.update_self:
527 updateSelf(fd, sys.argv[0])
528
529 # Maybe do nothing
530 if len(all_urls) < 1:
531 if not opts.update_self:
532 parser.error(u'you must provide at least one URL')
533 else:
534 sys.exit()
535
536 try:
537 retcode = fd.download(all_urls)
538 except MaxDownloadsReached:
539 fd.to_screen(u'--max-download limit reached, aborting.')
540 retcode = 101
541
542 # Dump cookie jar if requested
543 if opts.cookiefile is not None:
544 try:
545 jar.save()
546 except (IOError, OSError), err:
547 sys.exit(u'ERROR: unable to save cookie jar')
548
549 sys.exit(retcode)
550
551 def main():
552 try:
553 _real_main()
554 except DownloadError:
555 sys.exit(1)
556 except SameFileError:
557 sys.exit(u'ERROR: fixed output name but more than one file to download')
558 except KeyboardInterrupt:
559 sys.exit(u'\nERROR: Interrupted by user')