youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34 )
  35
  36 __license__ = 'Public Domain'
  37
  38 import codecs
  39 import collections
  40 import getpass
  41 import optparse
  42 import os
  43 import random
  44 import re
  45 import shlex
  46 import socket
  47 import subprocess
  48 import sys
  49 import warnings
  50 import platform
  51
  52
  53 from .utils import *
  54 from .update import update_self
  55 from .version import __version__
  56 from .FileDownloader import *
  57 from .extractor import gen_extractors
  58 from .YoutubeDL import YoutubeDL
  59 from .PostProcessor import *
  60
  61 def parseOpts(overrideArguments=None):
  62     def _readOptions(filename_bytes):
  63         try:
  64             optionf = open(filename_bytes)
  65         except IOError:
  66             return [] # silently skip if file is not present
  67         try:
  68             res = []
  69             for l in optionf:
  70                 res += shlex.split(l, comments=True)
  71         finally:
  72             optionf.close()
  73         return res
  74
  75     def _format_option_string(option):
  76         ''' ('-o', '--option') -> -o, --format METAVAR'''
  77
  78         opts = []
  79
  80         if option._short_opts:
  81             opts.append(option._short_opts[0])
  82         if option._long_opts:
  83             opts.append(option._long_opts[0])
  84         if len(opts) > 1:
  85             opts.insert(1, ', ')
  86
  87         if option.takes_value(): opts.append(' %s' % option.metavar)
  88
  89         return "".join(opts)
  90
  91     def _comma_separated_values_options_callback(option, opt_str, value, parser):
  92         setattr(parser.values, option.dest, value.split(','))
  93
  94     def _find_term_columns():
  95         columns = os.environ.get('COLUMNS', None)
  96         if columns:
  97             return int(columns)
  98
  99         try:
 100             sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 101             out,err = sp.communicate()
 102             return int(out.split()[1])
 103         except:
 104             pass
 105         return None
 106
 107     def _hide_login_info(opts):
 108         opts = list(opts)
 109         for private_opt in ['-p', '--password', '-u', '--username']:
 110             try:
 111                 i = opts.index(private_opt)
 112                 opts[i+1] = '<PRIVATE>'
 113             except ValueError:
 114                 pass
 115         return opts
 116
 117     max_width = 80
 118     max_help_position = 80
 119
 120     # No need to wrap help messages if we're on a wide console
 121     columns = _find_term_columns()
 122     if columns: max_width = columns
 123
 124     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 125     fmt.format_option_strings = _format_option_string
 126
 127     kw = {
 128         'version'   : __version__,
 129         'formatter' : fmt,
 130         'usage' : '%prog [options] url [url...]',
 131         'conflict_handler' : 'resolve',
 132     }
 133
 134     parser = optparse.OptionParser(**kw)
 135
 136     # option groups
 137     general        = optparse.OptionGroup(parser, 'General Options')
 138     selection      = optparse.OptionGroup(parser, 'Video Selection')
 139     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 140     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 141     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 142     downloader     = optparse.OptionGroup(parser, 'Download Options')
 143     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 144     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 145     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 146
 147     general.add_option('-h', '--help',
 148             action='help', help='print this help text and exit')
 149     general.add_option('-v', '--version',
 150             action='version', help='print program version and exit')
 151     general.add_option('-U', '--update',
 152             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 153     general.add_option('-i', '--ignore-errors',
 154             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 155     general.add_option('--dump-user-agent',
 156             action='store_true', dest='dump_user_agent',
 157             help='display the current browser identification', default=False)
 158     general.add_option('--user-agent',
 159             dest='user_agent', help='specify a custom user agent', metavar='UA')
 160     general.add_option('--referer',
 161             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 162             metavar='REF', default=None)
 163     general.add_option('--list-extractors',
 164             action='store_true', dest='list_extractors',
 165             help='List all supported extractors and the URLs they would handle', default=False)
 166     general.add_option('--extractor-descriptions',
 167             action='store_true', dest='list_extractor_descriptions',
 168             help='Output descriptions of all supported extractors', default=False)
 169     general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
 170     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 171     general.add_option(
 172         '--cache-dir', dest='cachedir', default=get_cachedir(),
 173         help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
 174     general.add_option(
 175         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 176         help='Disable filesystem caching')
 177
 178
 179     selection.add_option('--playlist-start',
 180             dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
 181     selection.add_option('--playlist-end',
 182             dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
 183     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 184     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 185     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
 186     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 187     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 188     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 189     selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
 190     selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 191     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 192     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 193                          help='download only videos suitable for the given age',
 194                          default=None, type=int)
 195     selection.add_option('--download-archive', metavar='FILE',
 196                          dest='download_archive',
 197                          help='Download only videos not present in the archive file. Record all downloaded videos in it.')
 198
 199
 200     authentication.add_option('-u', '--username',
 201             dest='username', metavar='USERNAME', help='account username')
 202     authentication.add_option('-p', '--password',
 203             dest='password', metavar='PASSWORD', help='account password')
 204     authentication.add_option('-n', '--netrc',
 205             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 206     authentication.add_option('--video-password',
 207             dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)')
 208
 209
 210     video_format.add_option('-f', '--format',
 211             action='store', dest='format', metavar='FORMAT',
 212             help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
 213     video_format.add_option('--all-formats',
 214             action='store_const', dest='format', help='download all available video formats', const='all')
 215     video_format.add_option('--prefer-free-formats',
 216             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 217     video_format.add_option('--max-quality',
 218             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 219     video_format.add_option('-F', '--list-formats',
 220             action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 221
 222     subtitles.add_option('--write-sub', '--write-srt',
 223             action='store_true', dest='writesubtitles',
 224             help='write subtitle file', default=False)
 225     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 226             action='store_true', dest='writeautomaticsub',
 227             help='write automatic subtitle file (youtube only)', default=False)
 228     subtitles.add_option('--all-subs',
 229             action='store_true', dest='allsubtitles',
 230             help='downloads all the available subtitles of the video', default=False)
 231     subtitles.add_option('--list-subs',
 232             action='store_true', dest='listsubtitles',
 233             help='lists all available subtitles for the video', default=False)
 234     subtitles.add_option('--sub-format',
 235             action='store', dest='subtitlesformat', metavar='FORMAT',
 236             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 237     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 238             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 239             default=[], callback=_comma_separated_values_options_callback,
 240             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 241
 242     downloader.add_option('-r', '--rate-limit',
 243             dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)')
 244     downloader.add_option('-R', '--retries',
 245             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 246     downloader.add_option('--buffer-size',
 247             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024")
 248     downloader.add_option('--no-resize-buffer',
 249             action='store_true', dest='noresizebuffer',
 250             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 251     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 252
 253     verbosity.add_option('-q', '--quiet',
 254             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 255     verbosity.add_option('-s', '--simulate',
 256             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 257     verbosity.add_option('--skip-download',
 258             action='store_true', dest='skip_download', help='do not download the video', default=False)
 259     verbosity.add_option('-g', '--get-url',
 260             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 261     verbosity.add_option('-e', '--get-title',
 262             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 263     verbosity.add_option('--get-id',
 264             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 265     verbosity.add_option('--get-thumbnail',
 266             action='store_true', dest='getthumbnail',
 267             help='simulate, quiet but print thumbnail URL', default=False)
 268     verbosity.add_option('--get-description',
 269             action='store_true', dest='getdescription',
 270             help='simulate, quiet but print video description', default=False)
 271     verbosity.add_option('--get-filename',
 272             action='store_true', dest='getfilename',
 273             help='simulate, quiet but print output filename', default=False)
 274     verbosity.add_option('--get-format',
 275             action='store_true', dest='getformat',
 276             help='simulate, quiet but print output format', default=False)
 277     verbosity.add_option('--newline',
 278             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 279     verbosity.add_option('--no-progress',
 280             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 281     verbosity.add_option('--console-title',
 282             action='store_true', dest='consoletitle',
 283             help='display progress in console titlebar', default=False)
 284     verbosity.add_option('-v', '--verbose',
 285             action='store_true', dest='verbose', help='print various debugging information', default=False)
 286     verbosity.add_option('--dump-intermediate-pages',
 287             action='store_true', dest='dump_intermediate_pages', default=False,
 288             help='print downloaded pages to debug problems(very verbose)')
 289     verbosity.add_option('--youtube-print-sig-code',
 290             action='store_true', dest='youtube_print_sig_code', default=False,
 291             help=optparse.SUPPRESS_HELP)
 292
 293
 294     filesystem.add_option('-t', '--title',
 295             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 296     filesystem.add_option('--id',
 297             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 298     filesystem.add_option('-l', '--literal',
 299             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 300     filesystem.add_option('-A', '--auto-number',
 301             action='store_true', dest='autonumber',
 302             help='number downloaded files starting from 00000', default=False)
 303     filesystem.add_option('-o', '--output',
 304             dest='outtmpl', metavar='TEMPLATE',
 305             help=('output filename template. Use %(title)s to get the title, '
 306                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 307                   '%(autonumber)s to get an automatically incremented number, '
 308                   '%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), '
 309                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 310                   '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
 311                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 312                   'Use - to output to stdout. Can also be used to download to a different directory, '
 313                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 314     filesystem.add_option('--autonumber-size',
 315             dest='autonumber_size', metavar='NUMBER',
 316             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given')
 317     filesystem.add_option('--restrict-filenames',
 318             action='store_true', dest='restrictfilenames',
 319             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 320     filesystem.add_option('-a', '--batch-file',
 321             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 322     filesystem.add_option('-w', '--no-overwrites',
 323             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 324     filesystem.add_option('-c', '--continue',
 325             action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
 326     filesystem.add_option('--no-continue',
 327             action='store_false', dest='continue_dl',
 328             help='do not resume partially downloaded files (restart from beginning)')
 329     filesystem.add_option('--cookies',
 330             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 331     filesystem.add_option('--no-part',
 332             action='store_true', dest='nopart', help='do not use .part files', default=False)
 333     filesystem.add_option('--no-mtime',
 334             action='store_false', dest='updatetime',
 335             help='do not use the Last-modified header to set the file modification time', default=True)
 336     filesystem.add_option('--write-description',
 337             action='store_true', dest='writedescription',
 338             help='write video description to a .description file', default=False)
 339     filesystem.add_option('--write-info-json',
 340             action='store_true', dest='writeinfojson',
 341             help='write video metadata to a .info.json file', default=False)
 342     filesystem.add_option('--write-annotations',
 343             action='store_true', dest='writeannotations',
 344             help='write video annotations to a .annotation file', default=False)
 345     filesystem.add_option('--write-thumbnail',
 346             action='store_true', dest='writethumbnail',
 347             help='write thumbnail image to disk', default=False)
 348
 349
 350     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 351             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 352     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 353             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 354     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 355             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 356     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 357             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 358     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 359             help='keeps the video file on disk after the post-processing; the video is erased by default')
 360     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 361             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 362     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 363             help='embed subtitles in the video (only for mp4 videos)')
 364     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 365             help='add metadata to the files')
 366
 367
 368     parser.add_option_group(general)
 369     parser.add_option_group(selection)
 370     parser.add_option_group(downloader)
 371     parser.add_option_group(filesystem)
 372     parser.add_option_group(verbosity)
 373     parser.add_option_group(video_format)
 374     parser.add_option_group(subtitles)
 375     parser.add_option_group(authentication)
 376     parser.add_option_group(postproc)
 377
 378     if overrideArguments is not None:
 379         opts, args = parser.parse_args(overrideArguments)
 380         if opts.verbose:
 381             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 382     else:
 383         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 384         if xdg_config_home:
 385             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 386             if not os.path.isfile(userConfFile):
 387                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 388         else:
 389             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 390             if not os.path.isfile(userConfFile):
 391                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 392         systemConf = _readOptions('/etc/youtube-dl.conf')
 393         userConf = _readOptions(userConfFile)
 394         commandLineConf = sys.argv[1:]
 395         argv = systemConf + userConf + commandLineConf
 396         opts, args = parser.parse_args(argv)
 397         if opts.verbose:
 398             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 399             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 400             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 401
 402     return parser, opts, args
 403
 404 def _real_main(argv=None):
 405     # Compatibility fixes for Windows
 406     if sys.platform == 'win32':
 407         # https://github.com/rg3/youtube-dl/issues/820
 408         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 409
 410     parser, opts, args = parseOpts(argv)
 411
 412     # Open appropriate CookieJar
 413     if opts.cookiefile is None:
 414         jar = compat_cookiejar.CookieJar()
 415     else:
 416         try:
 417             jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
 418             if os.access(opts.cookiefile, os.R_OK):
 419                 jar.load()
 420         except (IOError, OSError) as err:
 421             if opts.verbose:
 422                 traceback.print_exc()
 423             write_string(u'ERROR: unable to open cookie file\n')
 424             sys.exit(101)
 425     # Set user agent
 426     if opts.user_agent is not None:
 427         std_headers['User-Agent'] = opts.user_agent
 428
 429     # Set referer
 430     if opts.referer is not None:
 431         std_headers['Referer'] = opts.referer
 432
 433     # Dump user agent
 434     if opts.dump_user_agent:
 435         compat_print(std_headers['User-Agent'])
 436         sys.exit(0)
 437
 438     # Batch file verification
 439     batchurls = []
 440     if opts.batchfile is not None:
 441         try:
 442             if opts.batchfile == '-':
 443                 batchfd = sys.stdin
 444             else:
 445                 batchfd = open(opts.batchfile, 'r')
 446             batchurls = batchfd.readlines()
 447             batchurls = [x.strip() for x in batchurls]
 448             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 449             if opts.verbose:
 450                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 451         except IOError:
 452             sys.exit(u'ERROR: batch file could not be read')
 453     all_urls = batchurls + args
 454     all_urls = [url.strip() for url in all_urls]
 455
 456     opener = _setup_opener(jar=jar, opts=opts)
 457
 458     extractors = gen_extractors()
 459
 460     if opts.list_extractors:
 461         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 462             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 463             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 464             all_urls = [url for url in all_urls if url not in matchedUrls]
 465             for mu in matchedUrls:
 466                 compat_print(u'  ' + mu)
 467         sys.exit(0)
 468     if opts.list_extractor_descriptions:
 469         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 470             if not ie._WORKING:
 471                 continue
 472             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 473             if desc is False:
 474                 continue
 475             if hasattr(ie, 'SEARCH_KEY'):
 476                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 477                 _COUNTS = (u'', u'5', u'10', u'all')
 478                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 479             compat_print(desc)
 480         sys.exit(0)
 481
 482
 483     # Conflicting, missing and erroneous options
 484     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 485         parser.error(u'using .netrc conflicts with giving username/password')
 486     if opts.password is not None and opts.username is None:
 487         parser.error(u' account username missing\n')
 488     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 489         parser.error(u'using output template conflicts with using title, video ID or auto number')
 490     if opts.usetitle and opts.useid:
 491         parser.error(u'using title conflicts with using video ID')
 492     if opts.username is not None and opts.password is None:
 493         opts.password = getpass.getpass(u'Type account password and press return:')
 494     if opts.ratelimit is not None:
 495         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 496         if numeric_limit is None:
 497             parser.error(u'invalid rate limit specified')
 498         opts.ratelimit = numeric_limit
 499     if opts.min_filesize is not None:
 500         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 501         if numeric_limit is None:
 502             parser.error(u'invalid min_filesize specified')
 503         opts.min_filesize = numeric_limit
 504     if opts.max_filesize is not None:
 505         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 506         if numeric_limit is None:
 507             parser.error(u'invalid max_filesize specified')
 508         opts.max_filesize = numeric_limit
 509     if opts.retries is not None:
 510         try:
 511             opts.retries = int(opts.retries)
 512         except (TypeError, ValueError) as err:
 513             parser.error(u'invalid retry count specified')
 514     if opts.buffersize is not None:
 515         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 516         if numeric_buffersize is None:
 517             parser.error(u'invalid buffer size specified')
 518         opts.buffersize = numeric_buffersize
 519     try:
 520         opts.playliststart = int(opts.playliststart)
 521         if opts.playliststart <= 0:
 522             raise ValueError(u'Playlist start must be positive')
 523     except (TypeError, ValueError) as err:
 524         parser.error(u'invalid playlist start number specified')
 525     try:
 526         opts.playlistend = int(opts.playlistend)
 527         if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
 528             raise ValueError(u'Playlist end must be greater than playlist start')
 529     except (TypeError, ValueError) as err:
 530         parser.error(u'invalid playlist end number specified')
 531     if opts.extractaudio:
 532         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 533             parser.error(u'invalid audio format specified')
 534     if opts.audioquality:
 535         opts.audioquality = opts.audioquality.strip('k').strip('K')
 536         if not opts.audioquality.isdigit():
 537             parser.error(u'invalid audio quality specified')
 538     if opts.recodevideo is not None:
 539         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 540             parser.error(u'invalid video recode format specified')
 541     if opts.date is not None:
 542         date = DateRange.day(opts.date)
 543     else:
 544         date = DateRange(opts.dateafter, opts.datebefore)
 545
 546     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 547     # this was the old behaviour if only --all-sub was given.
 548     if opts.allsubtitles and (opts.writeautomaticsub == False):
 549         opts.writesubtitles = True
 550
 551     if sys.version_info < (3,):
 552         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 553         if opts.outtmpl is not None:
 554             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 555     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 556             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 557             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 558             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 559             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 560             or (opts.useid and u'%(id)s.%(ext)s')
 561             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 562             or u'%(title)s-%(id)s.%(ext)s')
 563     if '%(ext)s' not in outtmpl and opts.extractaudio:
 564         parser.error(u'Cannot download a video and extract audio into the same'
 565                      u' file! Use "%%(ext)s" instead of %r' %
 566                      determine_ext(outtmpl, u''))
 567
 568     # YoutubeDL
 569     ydl = YoutubeDL({
 570         'usenetrc': opts.usenetrc,
 571         'username': opts.username,
 572         'password': opts.password,
 573         'videopassword': opts.videopassword,
 574         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 575         'forceurl': opts.geturl,
 576         'forcetitle': opts.gettitle,
 577         'forceid': opts.getid,
 578         'forcethumbnail': opts.getthumbnail,
 579         'forcedescription': opts.getdescription,
 580         'forcefilename': opts.getfilename,
 581         'forceformat': opts.getformat,
 582         'simulate': opts.simulate,
 583         'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 584         'format': opts.format,
 585         'format_limit': opts.format_limit,
 586         'listformats': opts.listformats,
 587         'outtmpl': outtmpl,
 588         'autonumber_size': opts.autonumber_size,
 589         'restrictfilenames': opts.restrictfilenames,
 590         'ignoreerrors': opts.ignoreerrors,
 591         'ratelimit': opts.ratelimit,
 592         'nooverwrites': opts.nooverwrites,
 593         'retries': opts.retries,
 594         'buffersize': opts.buffersize,
 595         'noresizebuffer': opts.noresizebuffer,
 596         'continuedl': opts.continue_dl,
 597         'noprogress': opts.noprogress,
 598         'progress_with_newline': opts.progress_with_newline,
 599         'playliststart': opts.playliststart,
 600         'playlistend': opts.playlistend,
 601         'noplaylist': opts.noplaylist,
 602         'logtostderr': opts.outtmpl == '-',
 603         'consoletitle': opts.consoletitle,
 604         'nopart': opts.nopart,
 605         'updatetime': opts.updatetime,
 606         'writedescription': opts.writedescription,
 607         'writeannotations': opts.writeannotations,
 608         'writeinfojson': opts.writeinfojson,
 609         'writethumbnail': opts.writethumbnail,
 610         'writesubtitles': opts.writesubtitles,
 611         'writeautomaticsub': opts.writeautomaticsub,
 612         'allsubtitles': opts.allsubtitles,
 613         'listsubtitles': opts.listsubtitles,
 614         'subtitlesformat': opts.subtitlesformat,
 615         'subtitleslangs': opts.subtitleslangs,
 616         'matchtitle': decodeOption(opts.matchtitle),
 617         'rejecttitle': decodeOption(opts.rejecttitle),
 618         'max_downloads': opts.max_downloads,
 619         'prefer_free_formats': opts.prefer_free_formats,
 620         'verbose': opts.verbose,
 621         'dump_intermediate_pages': opts.dump_intermediate_pages,
 622         'test': opts.test,
 623         'keepvideo': opts.keepvideo,
 624         'min_filesize': opts.min_filesize,
 625         'max_filesize': opts.max_filesize,
 626         'daterange': date,
 627         'cachedir': opts.cachedir,
 628         'youtube_print_sig_code': opts.youtube_print_sig_code,
 629         'age_limit': opts.age_limit,
 630         'download_archive': opts.download_archive,
 631         })
 632
 633     if opts.verbose:
 634         write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
 635         try:
 636             sp = subprocess.Popen(
 637                 ['git', 'rev-parse', '--short', 'HEAD'],
 638                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 639                 cwd=os.path.dirname(os.path.abspath(__file__)))
 640             out, err = sp.communicate()
 641             out = out.decode().strip()
 642             if re.match('[0-9a-f]+', out):
 643                 write_string(u'[debug] Git HEAD: ' + out + u'\n')
 644         except:
 645             try:
 646                 sys.exc_clear()
 647             except:
 648                 pass
 649         write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
 650
 651         proxy_map = {}
 652         for handler in opener.handlers:
 653             if hasattr(handler, 'proxies'):
 654                 proxy_map.update(handler.proxies)
 655         write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
 656
 657     ydl.add_default_info_extractors()
 658
 659     # PostProcessors
 660     # Add the metadata pp first, the other pps will copy it
 661     if opts.addmetadata:
 662         ydl.add_post_processor(FFmpegMetadataPP())
 663     if opts.extractaudio:
 664         ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 665     if opts.recodevideo:
 666         ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 667     if opts.embedsubtitles:
 668         ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 669
 670     # Update version
 671     if opts.update_self:
 672         update_self(ydl.to_screen, opts.verbose)
 673
 674     # Maybe do nothing
 675     if len(all_urls) < 1:
 676         if not opts.update_self:
 677             parser.error(u'you must provide at least one URL')
 678         else:
 679             sys.exit()
 680
 681     try:
 682         retcode = ydl.download(all_urls)
 683     except MaxDownloadsReached:
 684         ydl.to_screen(u'--max-download limit reached, aborting.')
 685         retcode = 101
 686
 687     # Dump cookie jar if requested
 688     if opts.cookiefile is not None:
 689         try:
 690             jar.save()
 691         except (IOError, OSError) as err:
 692             sys.exit(u'ERROR: unable to save cookie jar')
 693
 694     sys.exit(retcode)
 695
 696
 697 def _setup_opener(jar=None, opts=None, timeout=300):
 698     if opts is None:
 699         FakeOptions = collections.namedtuple(
 700             'FakeOptions', ['proxy', 'no_check_certificate'])
 701         opts = FakeOptions(proxy=None, no_check_certificate=False)
 702
 703     cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
 704     if opts.proxy is not None:
 705         if opts.proxy == '':
 706             proxies = {}
 707         else:
 708             proxies = {'http': opts.proxy, 'https': opts.proxy}
 709     else:
 710         proxies = compat_urllib_request.getproxies()
 711         # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
 712         if 'http' in proxies and 'https' not in proxies:
 713             proxies['https'] = proxies['http']
 714     proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 715     https_handler = make_HTTPS_handler(opts)
 716     opener = compat_urllib_request.build_opener(
 717         https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
 718     # Delete the default user-agent header, which would otherwise apply in
 719     # cases where our custom HTTP handler doesn't come into play
 720     # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
 721     opener.addheaders = []
 722     compat_urllib_request.install_opener(opener)
 723     socket.setdefaulttimeout(timeout)
 724     return opener
 725
 726
 727 def main(argv=None):
 728     try:
 729         _real_main(argv)
 730     except DownloadError:
 731         sys.exit(1)
 732     except SameFileError:
 733         sys.exit(u'ERROR: fixed output name but more than one file to download')
 734     except KeyboardInterrupt:
 735         sys.exit(u'\nERROR: Interrupted by user')