youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70     'David Fabijan',
  71     'Sebastian Haas',
  72 )
  73
  74 __license__ = 'Public Domain'
  75
  76 import codecs
  77 import io
  78 import optparse
  79 import os
  80 import random
  81 import shlex
  82 import shutil
  83 import sys
  84
  85
  86 from .utils import (
  87     compat_getpass,
  88     compat_print,
  89     DateRange,
  90     DEFAULT_OUTTMPL,
  91     decodeOption,
  92     get_term_width,
  93     DownloadError,
  94     get_cachedir,
  95     MaxDownloadsReached,
  96     preferredencoding,
  97     read_batch_urls,
  98     SameFileError,
  99     setproctitle,
 100     std_headers,
 101     write_string,
 102 )
 103 from .update import update_self
 104 from .downloader import (
 105     FileDownloader,
 106 )
 107 from .extractor import gen_extractors
 108 from .version import __version__
 109 from .YoutubeDL import YoutubeDL
 110 from .postprocessor import (
 111     AtomicParsleyPP,
 112     FFmpegAudioFixPP,
 113     FFmpegMetadataPP,
 114     FFmpegVideoConvertor,
 115     FFmpegExtractAudioPP,
 116     FFmpegEmbedSubtitlePP,
 117     XAttrMetadataPP,
 118 )
 119
 120
 121 def parseOpts(overrideArguments=None):
 122     def _readOptions(filename_bytes, default=[]):
 123         try:
 124             optionf = open(filename_bytes)
 125         except IOError:
 126             return default  # silently skip if file is not present
 127         try:
 128             res = []
 129             for l in optionf:
 130                 res += shlex.split(l, comments=True)
 131         finally:
 132             optionf.close()
 133         return res
 134
 135     def _readUserConf():
 136         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 137         if xdg_config_home:
 138             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 139             if not os.path.isfile(userConfFile):
 140                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 141         else:
 142             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 143             if not os.path.isfile(userConfFile):
 144                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 145         userConf = _readOptions(userConfFile, None)
 146
 147         if userConf is None:
 148             appdata_dir = os.environ.get('appdata')
 149             if appdata_dir:
 150                 userConf = _readOptions(
 151                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 152                     default=None)
 153                 if userConf is None:
 154                     userConf = _readOptions(
 155                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 156                         default=None)
 157
 158         if userConf is None:
 159             userConf = _readOptions(
 160                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 161                 default=None)
 162         if userConf is None:
 163             userConf = _readOptions(
 164                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 165                 default=None)
 166
 167         if userConf is None:
 168             userConf = []
 169
 170         return userConf
 171
 172     def _format_option_string(option):
 173         ''' ('-o', '--option') -> -o, --format METAVAR'''
 174
 175         opts = []
 176
 177         if option._short_opts:
 178             opts.append(option._short_opts[0])
 179         if option._long_opts:
 180             opts.append(option._long_opts[0])
 181         if len(opts) > 1:
 182             opts.insert(1, ', ')
 183
 184         if option.takes_value(): opts.append(' %s' % option.metavar)
 185
 186         return "".join(opts)
 187
 188     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 189         setattr(parser.values, option.dest, value.split(','))
 190
 191     def _hide_login_info(opts):
 192         opts = list(opts)
 193         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 194             try:
 195                 i = opts.index(private_opt)
 196                 opts[i+1] = '<PRIVATE>'
 197             except ValueError:
 198                 pass
 199         return opts
 200
 201     max_width = 80
 202     max_help_position = 80
 203
 204     # No need to wrap help messages if we're on a wide console
 205     columns = get_term_width()
 206     if columns: max_width = columns
 207
 208     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 209     fmt.format_option_strings = _format_option_string
 210
 211     kw = {
 212         'version'   : __version__,
 213         'formatter' : fmt,
 214         'usage' : '%prog [options] url [url...]',
 215         'conflict_handler' : 'resolve',
 216     }
 217
 218     parser = optparse.OptionParser(**kw)
 219
 220     # option groups
 221     general        = optparse.OptionGroup(parser, 'General Options')
 222     selection      = optparse.OptionGroup(parser, 'Video Selection')
 223     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 224     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 225     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 226     downloader     = optparse.OptionGroup(parser, 'Download Options')
 227     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 228     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 229     workarounds    = optparse.OptionGroup(parser, 'Workarounds')
 230     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 231
 232     general.add_option('-h', '--help',
 233             action='help', help='print this help text and exit')
 234     general.add_option('-v', '--version',
 235             action='version', help='print program version and exit')
 236     general.add_option('-U', '--update',
 237             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 238     general.add_option('-i', '--ignore-errors',
 239             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 240     general.add_option('--abort-on-error',
 241             action='store_false', dest='ignoreerrors',
 242             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 243     general.add_option('--dump-user-agent',
 244             action='store_true', dest='dump_user_agent',
 245             help='display the current browser identification', default=False)
 246     general.add_option('--list-extractors',
 247             action='store_true', dest='list_extractors',
 248             help='List all supported extractors and the URLs they would handle', default=False)
 249     general.add_option('--extractor-descriptions',
 250             action='store_true', dest='list_extractor_descriptions',
 251             help='Output descriptions of all supported extractors', default=False)
 252     general.add_option(
 253         '--proxy', dest='proxy', default=None, metavar='URL',
 254         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 255     general.add_option(
 256         '--socket-timeout', dest='socket_timeout',
 257         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 258     general.add_option(
 259         '--default-search',
 260         dest='default_search', metavar='PREFIX',
 261         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
 262     general.add_option(
 263         '--ignore-config',
 264         action='store_true',
 265         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 266
 267     selection.add_option(
 268         '--playlist-start',
 269         dest='playliststart', metavar='NUMBER', default=1, type=int,
 270         help='playlist video to start at (default is %default)')
 271     selection.add_option(
 272         '--playlist-end',
 273         dest='playlistend', metavar='NUMBER', default=None, type=int,
 274         help='playlist video to end at (default is last)')
 275     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 276     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 277     selection.add_option('--max-downloads', metavar='NUMBER',
 278                          dest='max_downloads', type=int, default=None,
 279                          help='Abort after downloading NUMBER files')
 280     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 281     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 282     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 283     selection.add_option(
 284         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 285         help='download only videos uploaded on or before this date (i.e. inclusive)')
 286     selection.add_option(
 287         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 288         help='download only videos uploaded on or after this date (i.e. inclusive)')
 289     selection.add_option(
 290         '--min-views', metavar='COUNT', dest='min_views',
 291         default=None, type=int,
 292         help="Do not download any videos with less than COUNT views",)
 293     selection.add_option(
 294         '--max-views', metavar='COUNT', dest='max_views',
 295         default=None, type=int,
 296         help="Do not download any videos with more than COUNT views",)
 297     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 298     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 299                          help='download only videos suitable for the given age',
 300                          default=None, type=int)
 301     selection.add_option('--download-archive', metavar='FILE',
 302                          dest='download_archive',
 303                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 304     selection.add_option(
 305         '--include-ads', dest='include_ads',
 306         action='store_true',
 307         help='Download advertisements as well (experimental)')
 308     selection.add_option(
 309         '--youtube-include-dash-manifest', action='store_true',
 310         dest='youtube_include_dash_manifest', default=False,
 311         help='Try to download the DASH manifest on YouTube videos (experimental)')
 312
 313     authentication.add_option('-u', '--username',
 314             dest='username', metavar='USERNAME', help='account username')
 315     authentication.add_option('-p', '--password',
 316             dest='password', metavar='PASSWORD', help='account password')
 317     authentication.add_option('-n', '--netrc',
 318             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 319     authentication.add_option('--video-password',
 320             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 321
 322
 323     video_format.add_option('-f', '--format',
 324             action='store', dest='format', metavar='FORMAT', default=None,
 325             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 326     video_format.add_option('--all-formats',
 327             action='store_const', dest='format', help='download all available video formats', const='all')
 328     video_format.add_option('--prefer-free-formats',
 329             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 330     video_format.add_option('--max-quality',
 331             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 332     video_format.add_option('-F', '--list-formats',
 333             action='store_true', dest='listformats', help='list all available formats')
 334
 335     subtitles.add_option('--write-sub', '--write-srt',
 336             action='store_true', dest='writesubtitles',
 337             help='write subtitle file', default=False)
 338     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 339             action='store_true', dest='writeautomaticsub',
 340             help='write automatic subtitle file (youtube only)', default=False)
 341     subtitles.add_option('--all-subs',
 342             action='store_true', dest='allsubtitles',
 343             help='downloads all the available subtitles of the video', default=False)
 344     subtitles.add_option('--list-subs',
 345             action='store_true', dest='listsubtitles',
 346             help='lists all available subtitles for the video', default=False)
 347     subtitles.add_option('--sub-format',
 348             action='store', dest='subtitlesformat', metavar='FORMAT',
 349             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 350     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 351             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 352             default=[], callback=_comma_separated_values_options_callback,
 353             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 354
 355     downloader.add_option('-r', '--rate-limit',
 356             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 357     downloader.add_option('-R', '--retries',
 358             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 359     downloader.add_option('--buffer-size',
 360             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 361     downloader.add_option('--no-resize-buffer',
 362             action='store_true', dest='noresizebuffer',
 363             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 364     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 365
 366     workarounds.add_option(
 367         '--encoding', dest='encoding', metavar='ENCODING',
 368         help='Force the specified encoding (experimental)')
 369     workarounds.add_option(
 370         '--no-check-certificate', action='store_true',
 371         dest='no_check_certificate', default=False,
 372         help='Suppress HTTPS certificate validation.')
 373     workarounds.add_option(
 374         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 375         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 376     workarounds.add_option(
 377         '--user-agent', metavar='UA',
 378         dest='user_agent', help='specify a custom user agent')
 379     workarounds.add_option(
 380         '--referer', metavar='REF',
 381         dest='referer', default=None,
 382         help='specify a custom referer, use if the video access is restricted to one domain',
 383     )
 384     workarounds.add_option(
 385         '--add-header', metavar='FIELD:VALUE',
 386         dest='headers', action='append',
 387         help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
 388     )
 389     workarounds.add_option(
 390         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 391         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 392
 393     verbosity.add_option('-q', '--quiet',
 394             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 395     verbosity.add_option(
 396         '--no-warnings',
 397         dest='no_warnings', action='store_true', default=False,
 398         help='Ignore warnings')
 399     verbosity.add_option('-s', '--simulate',
 400             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 401     verbosity.add_option('--skip-download',
 402             action='store_true', dest='skip_download', help='do not download the video', default=False)
 403     verbosity.add_option('-g', '--get-url',
 404             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 405     verbosity.add_option('-e', '--get-title',
 406             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 407     verbosity.add_option('--get-id',
 408             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 409     verbosity.add_option('--get-thumbnail',
 410             action='store_true', dest='getthumbnail',
 411             help='simulate, quiet but print thumbnail URL', default=False)
 412     verbosity.add_option('--get-description',
 413             action='store_true', dest='getdescription',
 414             help='simulate, quiet but print video description', default=False)
 415     verbosity.add_option('--get-duration',
 416             action='store_true', dest='getduration',
 417             help='simulate, quiet but print video length', default=False)
 418     verbosity.add_option('--get-filename',
 419             action='store_true', dest='getfilename',
 420             help='simulate, quiet but print output filename', default=False)
 421     verbosity.add_option('--get-format',
 422             action='store_true', dest='getformat',
 423             help='simulate, quiet but print output format', default=False)
 424     verbosity.add_option('-j', '--dump-json',
 425             action='store_true', dest='dumpjson',
 426             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 427     verbosity.add_option('--newline',
 428             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 429     verbosity.add_option('--no-progress',
 430             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 431     verbosity.add_option('--console-title',
 432             action='store_true', dest='consoletitle',
 433             help='display progress in console titlebar', default=False)
 434     verbosity.add_option('-v', '--verbose',
 435             action='store_true', dest='verbose', help='print various debugging information', default=False)
 436     verbosity.add_option('--dump-intermediate-pages',
 437             action='store_true', dest='dump_intermediate_pages', default=False,
 438             help='print downloaded pages to debug problems (very verbose)')
 439     verbosity.add_option('--write-pages',
 440             action='store_true', dest='write_pages', default=False,
 441             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 442     verbosity.add_option('--youtube-print-sig-code',
 443             action='store_true', dest='youtube_print_sig_code', default=False,
 444             help=optparse.SUPPRESS_HELP)
 445     verbosity.add_option('--print-traffic',
 446             dest='debug_printtraffic', action='store_true', default=False,
 447             help='Display sent and read HTTP traffic')
 448
 449
 450     filesystem.add_option('-a', '--batch-file',
 451             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 452     filesystem.add_option('--id',
 453             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 454     filesystem.add_option('-A', '--auto-number',
 455             action='store_true', dest='autonumber',
 456             help='number downloaded files starting from 00000', default=False)
 457     filesystem.add_option('-o', '--output',
 458             dest='outtmpl', metavar='TEMPLATE',
 459             help=('output filename template. Use %(title)s to get the title, '
 460                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 461                   '%(autonumber)s to get an automatically incremented number, '
 462                   '%(ext)s for the filename extension, '
 463                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 464                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 465                   '%(upload_date)s for the upload date (YYYYMMDD), '
 466                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 467                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 468                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 469                   '%(height)s and %(width)s for the width and height of the video format. '
 470                   '%(resolution)s for a textual description of the resolution of the video format. '
 471                   'Use - to output to stdout. Can also be used to download to a different directory, '
 472                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 473     filesystem.add_option('--autonumber-size',
 474             dest='autonumber_size', metavar='NUMBER',
 475             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 476     filesystem.add_option('--restrict-filenames',
 477             action='store_true', dest='restrictfilenames',
 478             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 479     filesystem.add_option('-t', '--title',
 480             action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
 481     filesystem.add_option('-l', '--literal',
 482             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 483     filesystem.add_option('-w', '--no-overwrites',
 484             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 485     filesystem.add_option('-c', '--continue',
 486             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 487     filesystem.add_option('--no-continue',
 488             action='store_false', dest='continue_dl',
 489             help='do not resume partially downloaded files (restart from beginning)')
 490     filesystem.add_option('--no-part',
 491             action='store_true', dest='nopart', help='do not use .part files', default=False)
 492     filesystem.add_option('--no-mtime',
 493             action='store_false', dest='updatetime',
 494             help='do not use the Last-modified header to set the file modification time', default=True)
 495     filesystem.add_option('--write-description',
 496             action='store_true', dest='writedescription',
 497             help='write video description to a .description file', default=False)
 498     filesystem.add_option('--write-info-json',
 499             action='store_true', dest='writeinfojson',
 500             help='write video metadata to a .info.json file', default=False)
 501     filesystem.add_option('--write-annotations',
 502             action='store_true', dest='writeannotations',
 503             help='write video annotations to a .annotation file', default=False)
 504     filesystem.add_option('--write-thumbnail',
 505             action='store_true', dest='writethumbnail',
 506             help='write thumbnail image to disk', default=False)
 507     filesystem.add_option('--load-info',
 508             dest='load_info_filename', metavar='FILE',
 509             help='json file containing the video information (created with the "--write-json" option)')
 510     filesystem.add_option('--cookies',
 511             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 512     filesystem.add_option(
 513         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 514         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 515     filesystem.add_option(
 516         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 517         help='Disable filesystem caching')
 518     filesystem.add_option(
 519         '--rm-cache-dir', action='store_true', dest='rm_cachedir',
 520         help='Delete all filesystem cache files')
 521
 522
 523     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 524             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 525     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 526             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 527     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 528             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 529     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 530             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
 531     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 532             help='keeps the video file on disk after the post-processing; the video is erased by default')
 533     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 534             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 535     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 536             help='embed subtitles in the video (only for mp4 videos)')
 537     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 538             help='embed thumbnail in the audio as cover art')
 539     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 540             help='write metadata to the video file')
 541     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 542             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 543     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 544         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 545     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 546         help='Prefer ffmpeg over avconv for running the postprocessors')
 547
 548
 549     parser.add_option_group(general)
 550     parser.add_option_group(selection)
 551     parser.add_option_group(downloader)
 552     parser.add_option_group(filesystem)
 553     parser.add_option_group(verbosity)
 554     parser.add_option_group(workarounds)
 555     parser.add_option_group(video_format)
 556     parser.add_option_group(subtitles)
 557     parser.add_option_group(authentication)
 558     parser.add_option_group(postproc)
 559
 560     if overrideArguments is not None:
 561         opts, args = parser.parse_args(overrideArguments)
 562         if opts.verbose:
 563             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 564     else:
 565         commandLineConf = sys.argv[1:]
 566         if '--ignore-config' in commandLineConf:
 567             systemConf = []
 568             userConf = []
 569         else:
 570             systemConf = _readOptions('/etc/youtube-dl.conf')
 571             if '--ignore-config' in systemConf:
 572                 userConf = []
 573             else:
 574                 userConf = _readUserConf()
 575         argv = systemConf + userConf + commandLineConf
 576
 577         opts, args = parser.parse_args(argv)
 578         if opts.verbose:
 579             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 580             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 581             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 582
 583     return parser, opts, args
 584
 585
 586 def _real_main(argv=None):
 587     # Compatibility fixes for Windows
 588     if sys.platform == 'win32':
 589         # https://github.com/rg3/youtube-dl/issues/820
 590         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 591
 592     setproctitle(u'youtube-dl')
 593
 594     parser, opts, args = parseOpts(argv)
 595
 596     # Set user agent
 597     if opts.user_agent is not None:
 598         std_headers['User-Agent'] = opts.user_agent
 599
 600     # Set referer
 601     if opts.referer is not None:
 602         std_headers['Referer'] = opts.referer
 603
 604     # Custom HTTP headers
 605     if opts.headers is not None:
 606         for h in opts.headers:
 607             if h.find(':', 1) < 0:
 608                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 609             key, value = h.split(':', 2)
 610             if opts.verbose:
 611                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 612             std_headers[key] = value
 613
 614     # Dump user agent
 615     if opts.dump_user_agent:
 616         compat_print(std_headers['User-Agent'])
 617         sys.exit(0)
 618
 619     # Batch file verification
 620     batch_urls = []
 621     if opts.batchfile is not None:
 622         try:
 623             if opts.batchfile == '-':
 624                 batchfd = sys.stdin
 625             else:
 626                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 627             batch_urls = read_batch_urls(batchfd)
 628             if opts.verbose:
 629                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 630         except IOError:
 631             sys.exit(u'ERROR: batch file could not be read')
 632     all_urls = batch_urls + args
 633     all_urls = [url.strip() for url in all_urls]
 634     _enc = preferredencoding()
 635     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 636
 637     extractors = gen_extractors()
 638
 639     if opts.list_extractors:
 640         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 641             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 642             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 643             for mu in matchedUrls:
 644                 compat_print(u'  ' + mu)
 645         sys.exit(0)
 646     if opts.list_extractor_descriptions:
 647         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 648             if not ie._WORKING:
 649                 continue
 650             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 651             if desc is False:
 652                 continue
 653             if hasattr(ie, 'SEARCH_KEY'):
 654                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 655                 _COUNTS = (u'', u'5', u'10', u'all')
 656                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 657             compat_print(desc)
 658         sys.exit(0)
 659
 660
 661     # Conflicting, missing and erroneous options
 662     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 663         parser.error(u'using .netrc conflicts with giving username/password')
 664     if opts.password is not None and opts.username is None:
 665         parser.error(u'account username missing\n')
 666     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 667         parser.error(u'using output template conflicts with using title, video ID or auto number')
 668     if opts.usetitle and opts.useid:
 669         parser.error(u'using title conflicts with using video ID')
 670     if opts.username is not None and opts.password is None:
 671         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 672     if opts.ratelimit is not None:
 673         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 674         if numeric_limit is None:
 675             parser.error(u'invalid rate limit specified')
 676         opts.ratelimit = numeric_limit
 677     if opts.min_filesize is not None:
 678         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 679         if numeric_limit is None:
 680             parser.error(u'invalid min_filesize specified')
 681         opts.min_filesize = numeric_limit
 682     if opts.max_filesize is not None:
 683         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 684         if numeric_limit is None:
 685             parser.error(u'invalid max_filesize specified')
 686         opts.max_filesize = numeric_limit
 687     if opts.retries is not None:
 688         try:
 689             opts.retries = int(opts.retries)
 690         except (TypeError, ValueError):
 691             parser.error(u'invalid retry count specified')
 692     if opts.buffersize is not None:
 693         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 694         if numeric_buffersize is None:
 695             parser.error(u'invalid buffer size specified')
 696         opts.buffersize = numeric_buffersize
 697     if opts.playliststart <= 0:
 698         raise ValueError(u'Playlist start must be positive')
 699     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 700         raise ValueError(u'Playlist end must be greater than playlist start')
 701     if opts.extractaudio:
 702         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 703             parser.error(u'invalid audio format specified')
 704     if opts.audioquality:
 705         opts.audioquality = opts.audioquality.strip('k').strip('K')
 706         if not opts.audioquality.isdigit():
 707             parser.error(u'invalid audio quality specified')
 708     if opts.recodevideo is not None:
 709         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 710             parser.error(u'invalid video recode format specified')
 711     if opts.date is not None:
 712         date = DateRange.day(opts.date)
 713     else:
 714         date = DateRange(opts.dateafter, opts.datebefore)
 715     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 716         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 717
 718     # Do not download videos when there are audio-only formats
 719     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 720         opts.format = 'bestaudio/best'
 721
 722     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 723     # this was the old behaviour if only --all-sub was given.
 724     if opts.allsubtitles and (opts.writeautomaticsub == False):
 725         opts.writesubtitles = True
 726
 727     if sys.version_info < (3,):
 728         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 729         if opts.outtmpl is not None:
 730             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 731     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 732             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 733             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 734             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 735             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 736             or (opts.useid and u'%(id)s.%(ext)s')
 737             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 738             or DEFAULT_OUTTMPL)
 739     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 740         parser.error(u'Cannot download a video and extract audio into the same'
 741                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 742                      u' template'.format(outtmpl))
 743
 744     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 745     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 746
 747     ydl_opts = {
 748         'usenetrc': opts.usenetrc,
 749         'username': opts.username,
 750         'password': opts.password,
 751         'videopassword': opts.videopassword,
 752         'quiet': (opts.quiet or any_printing),
 753         'no_warnings': opts.no_warnings,
 754         'forceurl': opts.geturl,
 755         'forcetitle': opts.gettitle,
 756         'forceid': opts.getid,
 757         'forcethumbnail': opts.getthumbnail,
 758         'forcedescription': opts.getdescription,
 759         'forceduration': opts.getduration,
 760         'forcefilename': opts.getfilename,
 761         'forceformat': opts.getformat,
 762         'forcejson': opts.dumpjson,
 763         'simulate': opts.simulate,
 764         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 765         'format': opts.format,
 766         'format_limit': opts.format_limit,
 767         'listformats': opts.listformats,
 768         'outtmpl': outtmpl,
 769         'autonumber_size': opts.autonumber_size,
 770         'restrictfilenames': opts.restrictfilenames,
 771         'ignoreerrors': opts.ignoreerrors,
 772         'ratelimit': opts.ratelimit,
 773         'nooverwrites': opts.nooverwrites,
 774         'retries': opts.retries,
 775         'buffersize': opts.buffersize,
 776         'noresizebuffer': opts.noresizebuffer,
 777         'continuedl': opts.continue_dl,
 778         'noprogress': opts.noprogress,
 779         'progress_with_newline': opts.progress_with_newline,
 780         'playliststart': opts.playliststart,
 781         'playlistend': opts.playlistend,
 782         'noplaylist': opts.noplaylist,
 783         'logtostderr': opts.outtmpl == '-',
 784         'consoletitle': opts.consoletitle,
 785         'nopart': opts.nopart,
 786         'updatetime': opts.updatetime,
 787         'writedescription': opts.writedescription,
 788         'writeannotations': opts.writeannotations,
 789         'writeinfojson': opts.writeinfojson,
 790         'writethumbnail': opts.writethumbnail,
 791         'writesubtitles': opts.writesubtitles,
 792         'writeautomaticsub': opts.writeautomaticsub,
 793         'allsubtitles': opts.allsubtitles,
 794         'listsubtitles': opts.listsubtitles,
 795         'subtitlesformat': opts.subtitlesformat,
 796         'subtitleslangs': opts.subtitleslangs,
 797         'matchtitle': decodeOption(opts.matchtitle),
 798         'rejecttitle': decodeOption(opts.rejecttitle),
 799         'max_downloads': opts.max_downloads,
 800         'prefer_free_formats': opts.prefer_free_formats,
 801         'verbose': opts.verbose,
 802         'dump_intermediate_pages': opts.dump_intermediate_pages,
 803         'write_pages': opts.write_pages,
 804         'test': opts.test,
 805         'keepvideo': opts.keepvideo,
 806         'min_filesize': opts.min_filesize,
 807         'max_filesize': opts.max_filesize,
 808         'min_views': opts.min_views,
 809         'max_views': opts.max_views,
 810         'daterange': date,
 811         'cachedir': opts.cachedir,
 812         'youtube_print_sig_code': opts.youtube_print_sig_code,
 813         'age_limit': opts.age_limit,
 814         'download_archive': download_archive_fn,
 815         'cookiefile': opts.cookiefile,
 816         'nocheckcertificate': opts.no_check_certificate,
 817         'prefer_insecure': opts.prefer_insecure,
 818         'proxy': opts.proxy,
 819         'socket_timeout': opts.socket_timeout,
 820         'bidi_workaround': opts.bidi_workaround,
 821         'debug_printtraffic': opts.debug_printtraffic,
 822         'prefer_ffmpeg': opts.prefer_ffmpeg,
 823         'include_ads': opts.include_ads,
 824         'default_search': opts.default_search,
 825         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 826         'encoding': opts.encoding,
 827     }
 828
 829     with YoutubeDL(ydl_opts) as ydl:
 830         ydl.print_debug_header()
 831         ydl.add_default_info_extractors()
 832
 833         # PostProcessors
 834         # Add the metadata pp first, the other pps will copy it
 835         if opts.addmetadata:
 836             ydl.add_post_processor(FFmpegMetadataPP())
 837         if opts.extractaudio:
 838             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 839         if opts.recodevideo:
 840             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 841         if opts.embedsubtitles:
 842             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 843         if opts.xattrs:
 844             ydl.add_post_processor(XAttrMetadataPP())
 845         if opts.embedthumbnail:
 846             if not opts.addmetadata:
 847                 ydl.add_post_processor(FFmpegAudioFixPP())
 848             ydl.add_post_processor(AtomicParsleyPP())
 849
 850         # Update version
 851         if opts.update_self:
 852             update_self(ydl.to_screen, opts.verbose)
 853
 854         # Remove cache dir
 855         if opts.rm_cachedir:
 856             if opts.cachedir is None:
 857                 ydl.to_screen(u'No cache dir specified (Did you combine --no-cache-dir and --rm-cache-dir?)')
 858             else:
 859                 if ('.cache' not in opts.cachedir) or ('youtube-dl' not in opts.cachedir):
 860                     ydl.to_screen(u'Not removing directory %s - this does not look like a cache dir')
 861                     retcode = 141
 862                 else:
 863                     ydl.to_screen(
 864                         u'Removing cache dir %s .' % opts.cachedir,
 865                         skip_eol=True)
 866                     if os.path.exists(opts.cachedir):
 867                         ydl.to_screen(u'.', skip_eol=True)
 868                         shutil.rmtree(opts.cachedir)
 869                     ydl.to_screen(u'.')
 870
 871         # Maybe do nothing
 872         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 873             if not (opts.update_self or opts.rm_cachedir):
 874                 parser.error(u'you must provide at least one URL')
 875             else:
 876                 sys.exit()
 877
 878         try:
 879             if opts.load_info_filename is not None:
 880                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 881             else:
 882                 retcode = ydl.download(all_urls)
 883         except MaxDownloadsReached:
 884             ydl.to_screen(u'--max-download limit reached, aborting.')
 885             retcode = 101
 886
 887     sys.exit(retcode)
 888
 889
 890 def main(argv=None):
 891     try:
 892         _real_main(argv)
 893     except DownloadError:
 894         sys.exit(1)
 895     except SameFileError:
 896         sys.exit(u'ERROR: fixed output name but more than one file to download')
 897     except KeyboardInterrupt:
 898         sys.exit(u'\nERROR: Interrupted by user')