]> jfr.im git - yt-dlp.git/blob - youtube_dl/__init__.py
Convert all tabs to 4 spaces (PEP8)
[yt-dlp.git] / youtube_dl / __init__.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import with_statement
5
6 __authors__ = (
7 'Ricardo Garcia Gonzalez',
8 'Danny Colligan',
9 'Benjamin Johnson',
10 'Vasyl\' Vavrychuk',
11 'Witold Baryluk',
12 'Paweł Paprota',
13 'Gergely Imreh',
14 'Rogério Brito',
15 'Philipp Hagemeister',
16 'Sören Schulze',
17 'Kevin Ngo',
18 'Ori Avtalion',
19 'shizeeg',
20 'Filippo Valsorda',
21 'Christian Albrecht',
22 )
23
24 __license__ = 'Public Domain'
25 __version__ = '2012.11.29'
26
27 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
28 UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION'
29 UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe'
30
31
32 import getpass
33 import optparse
34 import os
35 import re
36 import shlex
37 import socket
38 import subprocess
39 import sys
40 import warnings
41
42 from utils import *
43 from FileDownloader import *
44 from InfoExtractors import *
45 from PostProcessor import *
46
47 def updateSelf(downloader, filename):
48 ''' Update the program file with the latest version from the repository '''
49 # Note: downloader only used for options
50
51 if not os.access(filename, os.W_OK):
52 sys.exit('ERROR: no write permissions on %s' % filename)
53
54 downloader.to_screen(u'Updating to latest version...')
55
56 urlv = compat_urllib_request.urlopen(UPDATE_URL_VERSION)
57 newversion = urlv.read().strip()
58 if newversion == __version__:
59 downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
60 return
61 urlv.close()
62
63 if hasattr(sys, "frozen"): #py2exe
64 exe = os.path.abspath(filename)
65 directory = os.path.dirname(exe)
66 if not os.access(directory, os.W_OK):
67 sys.exit('ERROR: no write permissions on %s' % directory)
68
69 try:
70 urlh = compat_urllib_request.urlopen(UPDATE_URL_EXE)
71 newcontent = urlh.read()
72 urlh.close()
73 with open(exe + '.new', 'wb') as outf:
74 outf.write(newcontent)
75 except (IOError, OSError) as err:
76 sys.exit('ERROR: unable to download latest version')
77
78 try:
79 bat = os.path.join(directory, 'youtube-dl-updater.bat')
80 b = open(bat, 'w')
81 b.write("""
82 echo Updating youtube-dl...
83 ping 127.0.0.1 -n 5 -w 1000 > NUL
84 move /Y "%s.new" "%s"
85 del "%s"
86 \n""" %(exe, exe, bat))
87 b.close()
88
89 os.startfile(bat)
90 except (IOError, OSError) as err:
91 sys.exit('ERROR: unable to overwrite current version')
92
93 else:
94 try:
95 urlh = compat_urllib_request.urlopen(UPDATE_URL)
96 newcontent = urlh.read()
97 urlh.close()
98 except (IOError, OSError) as err:
99 sys.exit('ERROR: unable to download latest version')
100
101 try:
102 with open(filename, 'wb') as outf:
103 outf.write(newcontent)
104 except (IOError, OSError) as err:
105 sys.exit('ERROR: unable to overwrite current version')
106
107 downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
108
109 def parseOpts():
110 def _readOptions(filename_bytes):
111 try:
112 optionf = open(filename_bytes)
113 except IOError:
114 return [] # silently skip if file is not present
115 try:
116 res = []
117 for l in optionf:
118 res += shlex.split(l, comments=True)
119 finally:
120 optionf.close()
121 return res
122
123 def _format_option_string(option):
124 ''' ('-o', '--option') -> -o, --format METAVAR'''
125
126 opts = []
127
128 if option._short_opts:
129 opts.append(option._short_opts[0])
130 if option._long_opts:
131 opts.append(option._long_opts[0])
132 if len(opts) > 1:
133 opts.insert(1, ', ')
134
135 if option.takes_value(): opts.append(' %s' % option.metavar)
136
137 return "".join(opts)
138
139 def _find_term_columns():
140 columns = os.environ.get('COLUMNS', None)
141 if columns:
142 return int(columns)
143
144 try:
145 sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
146 out,err = sp.communicate()
147 return int(out.split()[1])
148 except:
149 pass
150 return None
151
152 max_width = 80
153 max_help_position = 80
154
155 # No need to wrap help messages if we're on a wide console
156 columns = _find_term_columns()
157 if columns: max_width = columns
158
159 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
160 fmt.format_option_strings = _format_option_string
161
162 kw = {
163 'version' : __version__,
164 'formatter' : fmt,
165 'usage' : '%prog [options] url [url...]',
166 'conflict_handler' : 'resolve',
167 }
168
169 parser = optparse.OptionParser(**kw)
170
171 # option groups
172 general = optparse.OptionGroup(parser, 'General Options')
173 selection = optparse.OptionGroup(parser, 'Video Selection')
174 authentication = optparse.OptionGroup(parser, 'Authentication Options')
175 video_format = optparse.OptionGroup(parser, 'Video Format Options')
176 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
177 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
178 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
179
180 general.add_option('-h', '--help',
181 action='help', help='print this help text and exit')
182 general.add_option('-v', '--version',
183 action='version', help='print program version and exit')
184 general.add_option('-U', '--update',
185 action='store_true', dest='update_self', help='update this program to latest version')
186 general.add_option('-i', '--ignore-errors',
187 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
188 general.add_option('-r', '--rate-limit',
189 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
190 general.add_option('-R', '--retries',
191 dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
192 general.add_option('--buffer-size',
193 dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024")
194 general.add_option('--no-resize-buffer',
195 action='store_true', dest='noresizebuffer',
196 help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
197 general.add_option('--dump-user-agent',
198 action='store_true', dest='dump_user_agent',
199 help='display the current browser identification', default=False)
200 general.add_option('--user-agent',
201 dest='user_agent', help='specify a custom user agent', metavar='UA')
202 general.add_option('--list-extractors',
203 action='store_true', dest='list_extractors',
204 help='List all supported extractors and the URLs they would handle', default=False)
205
206 selection.add_option('--playlist-start',
207 dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
208 selection.add_option('--playlist-end',
209 dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
210 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
211 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
212 selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
213
214 authentication.add_option('-u', '--username',
215 dest='username', metavar='USERNAME', help='account username')
216 authentication.add_option('-p', '--password',
217 dest='password', metavar='PASSWORD', help='account password')
218 authentication.add_option('-n', '--netrc',
219 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
220
221
222 video_format.add_option('-f', '--format',
223 action='store', dest='format', metavar='FORMAT', help='video format code')
224 video_format.add_option('--all-formats',
225 action='store_const', dest='format', help='download all available video formats', const='all')
226 video_format.add_option('--prefer-free-formats',
227 action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
228 video_format.add_option('--max-quality',
229 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
230 video_format.add_option('-F', '--list-formats',
231 action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
232 video_format.add_option('--write-srt',
233 action='store_true', dest='writesubtitles',
234 help='write video closed captions to a .srt file (currently youtube only)', default=False)
235 video_format.add_option('--srt-lang',
236 action='store', dest='subtitleslang', metavar='LANG',
237 help='language of the closed captions to download (optional) use IETF language tags like \'en\'')
238
239
240 verbosity.add_option('-q', '--quiet',
241 action='store_true', dest='quiet', help='activates quiet mode', default=False)
242 verbosity.add_option('-s', '--simulate',
243 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
244 verbosity.add_option('--skip-download',
245 action='store_true', dest='skip_download', help='do not download the video', default=False)
246 verbosity.add_option('-g', '--get-url',
247 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
248 verbosity.add_option('-e', '--get-title',
249 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
250 verbosity.add_option('--get-thumbnail',
251 action='store_true', dest='getthumbnail',
252 help='simulate, quiet but print thumbnail URL', default=False)
253 verbosity.add_option('--get-description',
254 action='store_true', dest='getdescription',
255 help='simulate, quiet but print video description', default=False)
256 verbosity.add_option('--get-filename',
257 action='store_true', dest='getfilename',
258 help='simulate, quiet but print output filename', default=False)
259 verbosity.add_option('--get-format',
260 action='store_true', dest='getformat',
261 help='simulate, quiet but print output format', default=False)
262 verbosity.add_option('--no-progress',
263 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
264 verbosity.add_option('--console-title',
265 action='store_true', dest='consoletitle',
266 help='display progress in console titlebar', default=False)
267 verbosity.add_option('-v', '--verbose',
268 action='store_true', dest='verbose', help='print various debugging information', default=False)
269
270
271 filesystem.add_option('-t', '--title',
272 action='store_true', dest='usetitle', help='use title in file name', default=False)
273 filesystem.add_option('--id',
274 action='store_true', dest='useid', help='use video ID in file name', default=False)
275 filesystem.add_option('-l', '--literal',
276 action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
277 filesystem.add_option('-A', '--auto-number',
278 action='store_true', dest='autonumber',
279 help='number downloaded files starting from 00000', default=False)
280 filesystem.add_option('-o', '--output',
281 dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
282 filesystem.add_option('--restrict-filenames',
283 action='store_true', dest='restrictfilenames',
284 help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
285 filesystem.add_option('-a', '--batch-file',
286 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
287 filesystem.add_option('-w', '--no-overwrites',
288 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
289 filesystem.add_option('-c', '--continue',
290 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
291 filesystem.add_option('--no-continue',
292 action='store_false', dest='continue_dl',
293 help='do not resume partially downloaded files (restart from beginning)')
294 filesystem.add_option('--cookies',
295 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
296 filesystem.add_option('--no-part',
297 action='store_true', dest='nopart', help='do not use .part files', default=False)
298 filesystem.add_option('--no-mtime',
299 action='store_false', dest='updatetime',
300 help='do not use the Last-modified header to set the file modification time', default=True)
301 filesystem.add_option('--write-description',
302 action='store_true', dest='writedescription',
303 help='write video description to a .description file', default=False)
304 filesystem.add_option('--write-info-json',
305 action='store_true', dest='writeinfojson',
306 help='write video metadata to a .info.json file', default=False)
307
308
309 postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
310 help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
311 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
312 help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default')
313 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
314 help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
315 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
316 help='keeps the video file on disk after the post-processing; the video is erased by default')
317
318
319 parser.add_option_group(general)
320 parser.add_option_group(selection)
321 parser.add_option_group(filesystem)
322 parser.add_option_group(verbosity)
323 parser.add_option_group(video_format)
324 parser.add_option_group(authentication)
325 parser.add_option_group(postproc)
326
327 xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
328 if xdg_config_home:
329 userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
330 else:
331 userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
332 argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
333 opts, args = parser.parse_args(argv)
334
335 return parser, opts, args
336
337 def gen_extractors():
338 """ Return a list of an instance of every supported extractor.
339 The order does matter; the first extractor matched is the one handling the URL.
340 """
341 return [
342 YoutubePlaylistIE(),
343 YoutubeChannelIE(),
344 YoutubeUserIE(),
345 YoutubeSearchIE(),
346 YoutubeIE(),
347 MetacafeIE(),
348 DailymotionIE(),
349 GoogleIE(),
350 GoogleSearchIE(),
351 PhotobucketIE(),
352 YahooIE(),
353 YahooSearchIE(),
354 DepositFilesIE(),
355 FacebookIE(),
356 BlipTVUserIE(),
357 BlipTVIE(),
358 VimeoIE(),
359 MyVideoIE(),
360 ComedyCentralIE(),
361 EscapistIE(),
362 CollegeHumorIE(),
363 XVideosIE(),
364 SoundcloudIE(),
365 InfoQIE(),
366 MixcloudIE(),
367 StanfordOpenClassroomIE(),
368 MTVIE(),
369 YoukuIE(),
370 XNXXIE(),
371 GooglePlusIE(),
372 ArteTvIE(),
373 GenericIE()
374 ]
375
376 def _real_main():
377 parser, opts, args = parseOpts()
378
379 # Open appropriate CookieJar
380 if opts.cookiefile is None:
381 jar = compat_cookiejar.CookieJar()
382 else:
383 try:
384 jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
385 if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
386 jar.load()
387 except (IOError, OSError) as err:
388 sys.exit(u'ERROR: unable to open cookie file')
389 # Set user agent
390 if opts.user_agent is not None:
391 std_headers['User-Agent'] = opts.user_agent
392
393 # Dump user agent
394 if opts.dump_user_agent:
395 print(std_headers['User-Agent'])
396 sys.exit(0)
397
398 # Batch file verification
399 batchurls = []
400 if opts.batchfile is not None:
401 try:
402 if opts.batchfile == '-':
403 batchfd = sys.stdin
404 else:
405 batchfd = open(opts.batchfile, 'r')
406 batchurls = batchfd.readlines()
407 batchurls = [x.strip() for x in batchurls]
408 batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
409 except IOError:
410 sys.exit(u'ERROR: batch file could not be read')
411 all_urls = batchurls + args
412 all_urls = [url.strip() for url in all_urls]
413
414 # General configuration
415 cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
416 proxy_handler = compat_urllib_request.ProxyHandler()
417 opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
418 compat_urllib_request.install_opener(opener)
419 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
420
421 extractors = gen_extractors()
422
423 if opts.list_extractors:
424 for ie in extractors:
425 print(ie.IE_NAME)
426 matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
427 all_urls = filter(lambda url: url not in matchedUrls, all_urls)
428 for mu in matchedUrls:
429 print(u' ' + mu)
430 sys.exit(0)
431
432 # Conflicting, missing and erroneous options
433 if opts.usenetrc and (opts.username is not None or opts.password is not None):
434 parser.error(u'using .netrc conflicts with giving username/password')
435 if opts.password is not None and opts.username is None:
436 parser.error(u'account username missing')
437 if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
438 parser.error(u'using output template conflicts with using title, video ID or auto number')
439 if opts.usetitle and opts.useid:
440 parser.error(u'using title conflicts with using video ID')
441 if opts.username is not None and opts.password is None:
442 opts.password = getpass.getpass(u'Type account password and press return:')
443 if opts.ratelimit is not None:
444 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
445 if numeric_limit is None:
446 parser.error(u'invalid rate limit specified')
447 opts.ratelimit = numeric_limit
448 if opts.retries is not None:
449 try:
450 opts.retries = int(opts.retries)
451 except (TypeError, ValueError) as err:
452 parser.error(u'invalid retry count specified')
453 if opts.buffersize is not None:
454 numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
455 if numeric_buffersize is None:
456 parser.error(u'invalid buffer size specified')
457 opts.buffersize = numeric_buffersize
458 try:
459 opts.playliststart = int(opts.playliststart)
460 if opts.playliststart <= 0:
461 raise ValueError(u'Playlist start must be positive')
462 except (TypeError, ValueError) as err:
463 parser.error(u'invalid playlist start number specified')
464 try:
465 opts.playlistend = int(opts.playlistend)
466 if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
467 raise ValueError(u'Playlist end must be greater than playlist start')
468 except (TypeError, ValueError) as err:
469 parser.error(u'invalid playlist end number specified')
470 if opts.extractaudio:
471 if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
472 parser.error(u'invalid audio format specified')
473 if opts.audioquality:
474 opts.audioquality = opts.audioquality.strip('k').strip('K')
475 if not opts.audioquality.isdigit():
476 parser.error(u'invalid audio quality specified')
477
478 # File downloader
479 fd = FileDownloader({
480 'usenetrc': opts.usenetrc,
481 'username': opts.username,
482 'password': opts.password,
483 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
484 'forceurl': opts.geturl,
485 'forcetitle': opts.gettitle,
486 'forcethumbnail': opts.getthumbnail,
487 'forcedescription': opts.getdescription,
488 'forcefilename': opts.getfilename,
489 'forceformat': opts.getformat,
490 'simulate': opts.simulate,
491 'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
492 'format': opts.format,
493 'format_limit': opts.format_limit,
494 'listformats': opts.listformats,
495 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
496 or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
497 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
498 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
499 or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
500 or (opts.useid and u'%(id)s.%(ext)s')
501 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
502 or u'%(id)s.%(ext)s'),
503 'restrictfilenames': opts.restrictfilenames,
504 'ignoreerrors': opts.ignoreerrors,
505 'ratelimit': opts.ratelimit,
506 'nooverwrites': opts.nooverwrites,
507 'retries': opts.retries,
508 'buffersize': opts.buffersize,
509 'noresizebuffer': opts.noresizebuffer,
510 'continuedl': opts.continue_dl,
511 'noprogress': opts.noprogress,
512 'playliststart': opts.playliststart,
513 'playlistend': opts.playlistend,
514 'logtostderr': opts.outtmpl == '-',
515 'consoletitle': opts.consoletitle,
516 'nopart': opts.nopart,
517 'updatetime': opts.updatetime,
518 'writedescription': opts.writedescription,
519 'writeinfojson': opts.writeinfojson,
520 'writesubtitles': opts.writesubtitles,
521 'subtitleslang': opts.subtitleslang,
522 'matchtitle': opts.matchtitle,
523 'rejecttitle': opts.rejecttitle,
524 'max_downloads': opts.max_downloads,
525 'prefer_free_formats': opts.prefer_free_formats,
526 'verbose': opts.verbose,
527 })
528
529 if opts.verbose:
530 fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies))
531
532 for extractor in extractors:
533 fd.add_info_extractor(extractor)
534
535 # PostProcessors
536 if opts.extractaudio:
537 fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
538
539 # Update version
540 if opts.update_self:
541 updateSelf(fd, sys.argv[0])
542
543 # Maybe do nothing
544 if len(all_urls) < 1:
545 if not opts.update_self:
546 parser.error(u'you must provide at least one URL')
547 else:
548 sys.exit()
549
550 try:
551 retcode = fd.download(all_urls)
552 except MaxDownloadsReached:
553 fd.to_screen(u'--max-download limit reached, aborting.')
554 retcode = 101
555
556 # Dump cookie jar if requested
557 if opts.cookiefile is not None:
558 try:
559 jar.save()
560 except (IOError, OSError) as err:
561 sys.exit(u'ERROR: unable to save cookie jar')
562
563 sys.exit(retcode)
564
565 def main():
566 try:
567 _real_main()
568 except DownloadError:
569 sys.exit(1)
570 except SameFileError:
571 sys.exit(u'ERROR: fixed output name but more than one file to download')
572 except KeyboardInterrupt:
573 sys.exit(u'\nERROR: Interrupted by user')