youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70     'David Fabijan',
  71 )
  72
  73 __license__ = 'Public Domain'
  74
  75 import codecs
  76 import io
  77 import optparse
  78 import os
  79 import random
  80 import shlex
  81 import shutil
  82 import sys
  83
  84
  85 from .utils import (
  86     compat_getpass,
  87     compat_print,
  88     DateRange,
  89     DEFAULT_OUTTMPL,
  90     decodeOption,
  91     get_term_width,
  92     DownloadError,
  93     get_cachedir,
  94     MaxDownloadsReached,
  95     preferredencoding,
  96     read_batch_urls,
  97     SameFileError,
  98     setproctitle,
  99     std_headers,
 100     write_string,
 101 )
 102 from .update import update_self
 103 from .downloader import (
 104     FileDownloader,
 105 )
 106 from .extractor import gen_extractors
 107 from .version import __version__
 108 from .YoutubeDL import YoutubeDL
 109 from .postprocessor import (
 110     AtomicParsleyPP,
 111     FFmpegAudioFixPP,
 112     FFmpegMetadataPP,
 113     FFmpegVideoConvertor,
 114     FFmpegExtractAudioPP,
 115     FFmpegEmbedSubtitlePP,
 116     XAttrMetadataPP,
 117 )
 118
 119
 120 def parseOpts(overrideArguments=None):
 121     def _readOptions(filename_bytes, default=[]):
 122         try:
 123             optionf = open(filename_bytes)
 124         except IOError:
 125             return default  # silently skip if file is not present
 126         try:
 127             res = []
 128             for l in optionf:
 129                 res += shlex.split(l, comments=True)
 130         finally:
 131             optionf.close()
 132         return res
 133
 134     def _readUserConf():
 135         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 136         if xdg_config_home:
 137             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 138             if not os.path.isfile(userConfFile):
 139                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 140         else:
 141             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 142             if not os.path.isfile(userConfFile):
 143                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 144         userConf = _readOptions(userConfFile, None)
 145
 146         if userConf is None:
 147             appdata_dir = os.environ.get('appdata')
 148             if appdata_dir:
 149                 userConf = _readOptions(
 150                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 151                     default=None)
 152                 if userConf is None:
 153                     userConf = _readOptions(
 154                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 155                         default=None)
 156
 157         if userConf is None:
 158             userConf = _readOptions(
 159                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 160                 default=None)
 161         if userConf is None:
 162             userConf = _readOptions(
 163                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 164                 default=None)
 165
 166         if userConf is None:
 167             userConf = []
 168
 169         return userConf
 170
 171     def _format_option_string(option):
 172         ''' ('-o', '--option') -> -o, --format METAVAR'''
 173
 174         opts = []
 175
 176         if option._short_opts:
 177             opts.append(option._short_opts[0])
 178         if option._long_opts:
 179             opts.append(option._long_opts[0])
 180         if len(opts) > 1:
 181             opts.insert(1, ', ')
 182
 183         if option.takes_value(): opts.append(' %s' % option.metavar)
 184
 185         return "".join(opts)
 186
 187     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 188         setattr(parser.values, option.dest, value.split(','))
 189
 190     def _hide_login_info(opts):
 191         opts = list(opts)
 192         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 193             try:
 194                 i = opts.index(private_opt)
 195                 opts[i+1] = '<PRIVATE>'
 196             except ValueError:
 197                 pass
 198         return opts
 199
 200     max_width = 80
 201     max_help_position = 80
 202
 203     # No need to wrap help messages if we're on a wide console
 204     columns = get_term_width()
 205     if columns: max_width = columns
 206
 207     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 208     fmt.format_option_strings = _format_option_string
 209
 210     kw = {
 211         'version'   : __version__,
 212         'formatter' : fmt,
 213         'usage' : '%prog [options] url [url...]',
 214         'conflict_handler' : 'resolve',
 215     }
 216
 217     parser = optparse.OptionParser(**kw)
 218
 219     # option groups
 220     general        = optparse.OptionGroup(parser, 'General Options')
 221     selection      = optparse.OptionGroup(parser, 'Video Selection')
 222     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 223     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 224     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 225     downloader     = optparse.OptionGroup(parser, 'Download Options')
 226     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 227     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 228     workarounds    = optparse.OptionGroup(parser, 'Workarounds')
 229     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 230
 231     general.add_option('-h', '--help',
 232             action='help', help='print this help text and exit')
 233     general.add_option('-v', '--version',
 234             action='version', help='print program version and exit')
 235     general.add_option('-U', '--update',
 236             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 237     general.add_option('-i', '--ignore-errors',
 238             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 239     general.add_option('--abort-on-error',
 240             action='store_false', dest='ignoreerrors',
 241             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 242     general.add_option('--dump-user-agent',
 243             action='store_true', dest='dump_user_agent',
 244             help='display the current browser identification', default=False)
 245     general.add_option('--list-extractors',
 246             action='store_true', dest='list_extractors',
 247             help='List all supported extractors and the URLs they would handle', default=False)
 248     general.add_option('--extractor-descriptions',
 249             action='store_true', dest='list_extractor_descriptions',
 250             help='Output descriptions of all supported extractors', default=False)
 251     general.add_option(
 252         '--proxy', dest='proxy', default=None, metavar='URL',
 253         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 254     general.add_option(
 255         '--socket-timeout', dest='socket_timeout',
 256         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 257     general.add_option(
 258         '--default-search',
 259         dest='default_search', metavar='PREFIX',
 260         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
 261     general.add_option(
 262         '--ignore-config',
 263         action='store_true',
 264         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 265
 266     selection.add_option(
 267         '--playlist-start',
 268         dest='playliststart', metavar='NUMBER', default=1, type=int,
 269         help='playlist video to start at (default is %default)')
 270     selection.add_option(
 271         '--playlist-end',
 272         dest='playlistend', metavar='NUMBER', default=None, type=int,
 273         help='playlist video to end at (default is last)')
 274     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 275     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 276     selection.add_option('--max-downloads', metavar='NUMBER',
 277                          dest='max_downloads', type=int, default=None,
 278                          help='Abort after downloading NUMBER files')
 279     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 280     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 281     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 282     selection.add_option(
 283         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 284         help='download only videos uploaded on or before this date (i.e. inclusive)')
 285     selection.add_option(
 286         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 287         help='download only videos uploaded on or after this date (i.e. inclusive)')
 288     selection.add_option(
 289         '--min-views', metavar='COUNT', dest='min_views',
 290         default=None, type=int,
 291         help="Do not download any videos with less than COUNT views",)
 292     selection.add_option(
 293         '--max-views', metavar='COUNT', dest='max_views',
 294         default=None, type=int,
 295         help="Do not download any videos with more than COUNT views",)
 296     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 297     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 298                          help='download only videos suitable for the given age',
 299                          default=None, type=int)
 300     selection.add_option('--download-archive', metavar='FILE',
 301                          dest='download_archive',
 302                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 303     selection.add_option(
 304         '--include-ads', dest='include_ads',
 305         action='store_true',
 306         help='Download advertisements as well (experimental)')
 307     selection.add_option(
 308         '--youtube-include-dash-manifest', action='store_true',
 309         dest='youtube_include_dash_manifest', default=False,
 310         help='Try to download the DASH manifest on YouTube videos (experimental)')
 311
 312     authentication.add_option('-u', '--username',
 313             dest='username', metavar='USERNAME', help='account username')
 314     authentication.add_option('-p', '--password',
 315             dest='password', metavar='PASSWORD', help='account password')
 316     authentication.add_option('-n', '--netrc',
 317             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 318     authentication.add_option('--video-password',
 319             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 320
 321
 322     video_format.add_option('-f', '--format',
 323             action='store', dest='format', metavar='FORMAT', default=None,
 324             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 325     video_format.add_option('--all-formats',
 326             action='store_const', dest='format', help='download all available video formats', const='all')
 327     video_format.add_option('--prefer-free-formats',
 328             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 329     video_format.add_option('--max-quality',
 330             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 331     video_format.add_option('-F', '--list-formats',
 332             action='store_true', dest='listformats', help='list all available formats')
 333
 334     subtitles.add_option('--write-sub', '--write-srt',
 335             action='store_true', dest='writesubtitles',
 336             help='write subtitle file', default=False)
 337     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 338             action='store_true', dest='writeautomaticsub',
 339             help='write automatic subtitle file (youtube only)', default=False)
 340     subtitles.add_option('--all-subs',
 341             action='store_true', dest='allsubtitles',
 342             help='downloads all the available subtitles of the video', default=False)
 343     subtitles.add_option('--list-subs',
 344             action='store_true', dest='listsubtitles',
 345             help='lists all available subtitles for the video', default=False)
 346     subtitles.add_option('--sub-format',
 347             action='store', dest='subtitlesformat', metavar='FORMAT',
 348             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 349     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 350             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 351             default=[], callback=_comma_separated_values_options_callback,
 352             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 353
 354     downloader.add_option('-r', '--rate-limit',
 355             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 356     downloader.add_option('-R', '--retries',
 357             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 358     downloader.add_option('--buffer-size',
 359             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 360     downloader.add_option('--no-resize-buffer',
 361             action='store_true', dest='noresizebuffer',
 362             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 363     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 364
 365     workarounds.add_option(
 366         '--encoding', dest='encoding', metavar='ENCODING',
 367         help='Force the specified encoding (experimental)')
 368     workarounds.add_option(
 369         '--no-check-certificate', action='store_true',
 370         dest='no_check_certificate', default=False,
 371         help='Suppress HTTPS certificate validation.')
 372     workarounds.add_option(
 373         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 374         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 375     workarounds.add_option(
 376         '--user-agent', metavar='UA',
 377         dest='user_agent', help='specify a custom user agent')
 378     workarounds.add_option(
 379         '--referer', metavar='REF',
 380         dest='referer', default=None,
 381         help='specify a custom referer, use if the video access is restricted to one domain',
 382     )
 383     workarounds.add_option(
 384         '--add-header', metavar='FIELD:VALUE',
 385         dest='headers', action='append',
 386         help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
 387     )
 388     workarounds.add_option(
 389         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 390         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 391
 392     verbosity.add_option('-q', '--quiet',
 393             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 394     verbosity.add_option(
 395         '--no-warnings',
 396         dest='no_warnings', action='store_true', default=False,
 397         help='Ignore warnings')
 398     verbosity.add_option('-s', '--simulate',
 399             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 400     verbosity.add_option('--skip-download',
 401             action='store_true', dest='skip_download', help='do not download the video', default=False)
 402     verbosity.add_option('-g', '--get-url',
 403             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 404     verbosity.add_option('-e', '--get-title',
 405             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 406     verbosity.add_option('--get-id',
 407             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 408     verbosity.add_option('--get-thumbnail',
 409             action='store_true', dest='getthumbnail',
 410             help='simulate, quiet but print thumbnail URL', default=False)
 411     verbosity.add_option('--get-description',
 412             action='store_true', dest='getdescription',
 413             help='simulate, quiet but print video description', default=False)
 414     verbosity.add_option('--get-duration',
 415             action='store_true', dest='getduration',
 416             help='simulate, quiet but print video length', default=False)
 417     verbosity.add_option('--get-filename',
 418             action='store_true', dest='getfilename',
 419             help='simulate, quiet but print output filename', default=False)
 420     verbosity.add_option('--get-format',
 421             action='store_true', dest='getformat',
 422             help='simulate, quiet but print output format', default=False)
 423     verbosity.add_option('-j', '--dump-json',
 424             action='store_true', dest='dumpjson',
 425             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 426     verbosity.add_option('--newline',
 427             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 428     verbosity.add_option('--no-progress',
 429             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 430     verbosity.add_option('--console-title',
 431             action='store_true', dest='consoletitle',
 432             help='display progress in console titlebar', default=False)
 433     verbosity.add_option('-v', '--verbose',
 434             action='store_true', dest='verbose', help='print various debugging information', default=False)
 435     verbosity.add_option('--dump-intermediate-pages',
 436             action='store_true', dest='dump_intermediate_pages', default=False,
 437             help='print downloaded pages to debug problems (very verbose)')
 438     verbosity.add_option('--write-pages',
 439             action='store_true', dest='write_pages', default=False,
 440             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 441     verbosity.add_option('--youtube-print-sig-code',
 442             action='store_true', dest='youtube_print_sig_code', default=False,
 443             help=optparse.SUPPRESS_HELP)
 444     verbosity.add_option('--print-traffic',
 445             dest='debug_printtraffic', action='store_true', default=False,
 446             help='Display sent and read HTTP traffic')
 447
 448
 449     filesystem.add_option('-a', '--batch-file',
 450             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 451     filesystem.add_option('--id',
 452             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 453     filesystem.add_option('-A', '--auto-number',
 454             action='store_true', dest='autonumber',
 455             help='number downloaded files starting from 00000', default=False)
 456     filesystem.add_option('-o', '--output',
 457             dest='outtmpl', metavar='TEMPLATE',
 458             help=('output filename template. Use %(title)s to get the title, '
 459                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 460                   '%(autonumber)s to get an automatically incremented number, '
 461                   '%(ext)s for the filename extension, '
 462                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 463                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 464                   '%(upload_date)s for the upload date (YYYYMMDD), '
 465                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 466                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 467                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 468                   '%(height)s and %(width)s for the width and height of the video format. '
 469                   '%(resolution)s for a textual description of the resolution of the video format. '
 470                   'Use - to output to stdout. Can also be used to download to a different directory, '
 471                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 472     filesystem.add_option('--autonumber-size',
 473             dest='autonumber_size', metavar='NUMBER',
 474             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 475     filesystem.add_option('--restrict-filenames',
 476             action='store_true', dest='restrictfilenames',
 477             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 478     filesystem.add_option('-t', '--title',
 479             action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
 480     filesystem.add_option('-l', '--literal',
 481             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 482     filesystem.add_option('-w', '--no-overwrites',
 483             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 484     filesystem.add_option('-c', '--continue',
 485             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 486     filesystem.add_option('--no-continue',
 487             action='store_false', dest='continue_dl',
 488             help='do not resume partially downloaded files (restart from beginning)')
 489     filesystem.add_option('--no-part',
 490             action='store_true', dest='nopart', help='do not use .part files', default=False)
 491     filesystem.add_option('--no-mtime',
 492             action='store_false', dest='updatetime',
 493             help='do not use the Last-modified header to set the file modification time', default=True)
 494     filesystem.add_option('--write-description',
 495             action='store_true', dest='writedescription',
 496             help='write video description to a .description file', default=False)
 497     filesystem.add_option('--write-info-json',
 498             action='store_true', dest='writeinfojson',
 499             help='write video metadata to a .info.json file', default=False)
 500     filesystem.add_option('--write-annotations',
 501             action='store_true', dest='writeannotations',
 502             help='write video annotations to a .annotation file', default=False)
 503     filesystem.add_option('--write-thumbnail',
 504             action='store_true', dest='writethumbnail',
 505             help='write thumbnail image to disk', default=False)
 506     filesystem.add_option('--load-info',
 507             dest='load_info_filename', metavar='FILE',
 508             help='json file containing the video information (created with the "--write-json" option)')
 509     filesystem.add_option('--cookies',
 510             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 511     filesystem.add_option(
 512         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 513         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 514     filesystem.add_option(
 515         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 516         help='Disable filesystem caching')
 517     filesystem.add_option(
 518         '--rm-cache-dir', action='store_true', dest='rm_cachedir',
 519         help='Delete all filesystem cache files')
 520
 521
 522     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 523             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 524     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 525             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 526     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 527             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 528     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 529             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
 530     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 531             help='keeps the video file on disk after the post-processing; the video is erased by default')
 532     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 533             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 534     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 535             help='embed subtitles in the video (only for mp4 videos)')
 536     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 537             help='embed thumbnail in the audio as cover art')
 538     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 539             help='write metadata to the video file')
 540     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 541             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 542     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 543         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 544     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 545         help='Prefer ffmpeg over avconv for running the postprocessors')
 546
 547
 548     parser.add_option_group(general)
 549     parser.add_option_group(selection)
 550     parser.add_option_group(downloader)
 551     parser.add_option_group(filesystem)
 552     parser.add_option_group(verbosity)
 553     parser.add_option_group(workarounds)
 554     parser.add_option_group(video_format)
 555     parser.add_option_group(subtitles)
 556     parser.add_option_group(authentication)
 557     parser.add_option_group(postproc)
 558
 559     if overrideArguments is not None:
 560         opts, args = parser.parse_args(overrideArguments)
 561         if opts.verbose:
 562             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 563     else:
 564         commandLineConf = sys.argv[1:]
 565         if '--ignore-config' in commandLineConf:
 566             systemConf = []
 567             userConf = []
 568         else:
 569             systemConf = _readOptions('/etc/youtube-dl.conf')
 570             if '--ignore-config' in systemConf:
 571                 userConf = []
 572             else:
 573                 userConf = _readUserConf()
 574         argv = systemConf + userConf + commandLineConf
 575
 576         opts, args = parser.parse_args(argv)
 577         if opts.verbose:
 578             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 579             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 580             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 581
 582     return parser, opts, args
 583
 584
 585 def _real_main(argv=None):
 586     # Compatibility fixes for Windows
 587     if sys.platform == 'win32':
 588         # https://github.com/rg3/youtube-dl/issues/820
 589         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 590
 591     setproctitle(u'youtube-dl')
 592
 593     parser, opts, args = parseOpts(argv)
 594
 595     # Set user agent
 596     if opts.user_agent is not None:
 597         std_headers['User-Agent'] = opts.user_agent
 598
 599     # Set referer
 600     if opts.referer is not None:
 601         std_headers['Referer'] = opts.referer
 602
 603     # Custom HTTP headers
 604     if opts.headers is not None:
 605         for h in opts.headers:
 606             if h.find(':', 1) < 0:
 607                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 608             key, value = h.split(':', 2)
 609             if opts.verbose:
 610                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 611             std_headers[key] = value
 612
 613     # Dump user agent
 614     if opts.dump_user_agent:
 615         compat_print(std_headers['User-Agent'])
 616         sys.exit(0)
 617
 618     # Batch file verification
 619     batch_urls = []
 620     if opts.batchfile is not None:
 621         try:
 622             if opts.batchfile == '-':
 623                 batchfd = sys.stdin
 624             else:
 625                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 626             batch_urls = read_batch_urls(batchfd)
 627             if opts.verbose:
 628                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 629         except IOError:
 630             sys.exit(u'ERROR: batch file could not be read')
 631     all_urls = batch_urls + args
 632     all_urls = [url.strip() for url in all_urls]
 633     _enc = preferredencoding()
 634     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 635
 636     extractors = gen_extractors()
 637
 638     if opts.list_extractors:
 639         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 640             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 641             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 642             for mu in matchedUrls:
 643                 compat_print(u'  ' + mu)
 644         sys.exit(0)
 645     if opts.list_extractor_descriptions:
 646         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 647             if not ie._WORKING:
 648                 continue
 649             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 650             if desc is False:
 651                 continue
 652             if hasattr(ie, 'SEARCH_KEY'):
 653                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 654                 _COUNTS = (u'', u'5', u'10', u'all')
 655                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 656             compat_print(desc)
 657         sys.exit(0)
 658
 659
 660     # Conflicting, missing and erroneous options
 661     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 662         parser.error(u'using .netrc conflicts with giving username/password')
 663     if opts.password is not None and opts.username is None:
 664         parser.error(u'account username missing\n')
 665     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 666         parser.error(u'using output template conflicts with using title, video ID or auto number')
 667     if opts.usetitle and opts.useid:
 668         parser.error(u'using title conflicts with using video ID')
 669     if opts.username is not None and opts.password is None:
 670         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 671     if opts.ratelimit is not None:
 672         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 673         if numeric_limit is None:
 674             parser.error(u'invalid rate limit specified')
 675         opts.ratelimit = numeric_limit
 676     if opts.min_filesize is not None:
 677         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 678         if numeric_limit is None:
 679             parser.error(u'invalid min_filesize specified')
 680         opts.min_filesize = numeric_limit
 681     if opts.max_filesize is not None:
 682         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 683         if numeric_limit is None:
 684             parser.error(u'invalid max_filesize specified')
 685         opts.max_filesize = numeric_limit
 686     if opts.retries is not None:
 687         try:
 688             opts.retries = int(opts.retries)
 689         except (TypeError, ValueError):
 690             parser.error(u'invalid retry count specified')
 691     if opts.buffersize is not None:
 692         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 693         if numeric_buffersize is None:
 694             parser.error(u'invalid buffer size specified')
 695         opts.buffersize = numeric_buffersize
 696     if opts.playliststart <= 0:
 697         raise ValueError(u'Playlist start must be positive')
 698     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 699         raise ValueError(u'Playlist end must be greater than playlist start')
 700     if opts.extractaudio:
 701         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 702             parser.error(u'invalid audio format specified')
 703     if opts.audioquality:
 704         opts.audioquality = opts.audioquality.strip('k').strip('K')
 705         if not opts.audioquality.isdigit():
 706             parser.error(u'invalid audio quality specified')
 707     if opts.recodevideo is not None:
 708         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 709             parser.error(u'invalid video recode format specified')
 710     if opts.date is not None:
 711         date = DateRange.day(opts.date)
 712     else:
 713         date = DateRange(opts.dateafter, opts.datebefore)
 714     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 715         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 716
 717     # Do not download videos when there are audio-only formats
 718     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 719         opts.format = 'bestaudio/best'
 720
 721     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 722     # this was the old behaviour if only --all-sub was given.
 723     if opts.allsubtitles and (opts.writeautomaticsub == False):
 724         opts.writesubtitles = True
 725
 726     if sys.version_info < (3,):
 727         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 728         if opts.outtmpl is not None:
 729             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 730     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 731             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 732             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 733             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 734             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 735             or (opts.useid and u'%(id)s.%(ext)s')
 736             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 737             or DEFAULT_OUTTMPL)
 738     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 739         parser.error(u'Cannot download a video and extract audio into the same'
 740                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 741                      u' template'.format(outtmpl))
 742
 743     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 744     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 745
 746     ydl_opts = {
 747         'usenetrc': opts.usenetrc,
 748         'username': opts.username,
 749         'password': opts.password,
 750         'videopassword': opts.videopassword,
 751         'quiet': (opts.quiet or any_printing),
 752         'no_warnings': opts.no_warnings,
 753         'forceurl': opts.geturl,
 754         'forcetitle': opts.gettitle,
 755         'forceid': opts.getid,
 756         'forcethumbnail': opts.getthumbnail,
 757         'forcedescription': opts.getdescription,
 758         'forceduration': opts.getduration,
 759         'forcefilename': opts.getfilename,
 760         'forceformat': opts.getformat,
 761         'forcejson': opts.dumpjson,
 762         'simulate': opts.simulate,
 763         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 764         'format': opts.format,
 765         'format_limit': opts.format_limit,
 766         'listformats': opts.listformats,
 767         'outtmpl': outtmpl,
 768         'autonumber_size': opts.autonumber_size,
 769         'restrictfilenames': opts.restrictfilenames,
 770         'ignoreerrors': opts.ignoreerrors,
 771         'ratelimit': opts.ratelimit,
 772         'nooverwrites': opts.nooverwrites,
 773         'retries': opts.retries,
 774         'buffersize': opts.buffersize,
 775         'noresizebuffer': opts.noresizebuffer,
 776         'continuedl': opts.continue_dl,
 777         'noprogress': opts.noprogress,
 778         'progress_with_newline': opts.progress_with_newline,
 779         'playliststart': opts.playliststart,
 780         'playlistend': opts.playlistend,
 781         'noplaylist': opts.noplaylist,
 782         'logtostderr': opts.outtmpl == '-',
 783         'consoletitle': opts.consoletitle,
 784         'nopart': opts.nopart,
 785         'updatetime': opts.updatetime,
 786         'writedescription': opts.writedescription,
 787         'writeannotations': opts.writeannotations,
 788         'writeinfojson': opts.writeinfojson,
 789         'writethumbnail': opts.writethumbnail,
 790         'writesubtitles': opts.writesubtitles,
 791         'writeautomaticsub': opts.writeautomaticsub,
 792         'allsubtitles': opts.allsubtitles,
 793         'listsubtitles': opts.listsubtitles,
 794         'subtitlesformat': opts.subtitlesformat,
 795         'subtitleslangs': opts.subtitleslangs,
 796         'matchtitle': decodeOption(opts.matchtitle),
 797         'rejecttitle': decodeOption(opts.rejecttitle),
 798         'max_downloads': opts.max_downloads,
 799         'prefer_free_formats': opts.prefer_free_formats,
 800         'verbose': opts.verbose,
 801         'dump_intermediate_pages': opts.dump_intermediate_pages,
 802         'write_pages': opts.write_pages,
 803         'test': opts.test,
 804         'keepvideo': opts.keepvideo,
 805         'min_filesize': opts.min_filesize,
 806         'max_filesize': opts.max_filesize,
 807         'min_views': opts.min_views,
 808         'max_views': opts.max_views,
 809         'daterange': date,
 810         'cachedir': opts.cachedir,
 811         'youtube_print_sig_code': opts.youtube_print_sig_code,
 812         'age_limit': opts.age_limit,
 813         'download_archive': download_archive_fn,
 814         'cookiefile': opts.cookiefile,
 815         'nocheckcertificate': opts.no_check_certificate,
 816         'prefer_insecure': opts.prefer_insecure,
 817         'proxy': opts.proxy,
 818         'socket_timeout': opts.socket_timeout,
 819         'bidi_workaround': opts.bidi_workaround,
 820         'debug_printtraffic': opts.debug_printtraffic,
 821         'prefer_ffmpeg': opts.prefer_ffmpeg,
 822         'include_ads': opts.include_ads,
 823         'default_search': opts.default_search,
 824         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 825         'encoding': opts.encoding,
 826     }
 827
 828     with YoutubeDL(ydl_opts) as ydl:
 829         ydl.print_debug_header()
 830         ydl.add_default_info_extractors()
 831
 832         # PostProcessors
 833         # Add the metadata pp first, the other pps will copy it
 834         if opts.addmetadata:
 835             ydl.add_post_processor(FFmpegMetadataPP())
 836         if opts.extractaudio:
 837             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 838         if opts.recodevideo:
 839             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 840         if opts.embedsubtitles:
 841             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 842         if opts.xattrs:
 843             ydl.add_post_processor(XAttrMetadataPP())
 844         if opts.embedthumbnail:
 845             if not opts.addmetadata:
 846                 ydl.add_post_processor(FFmpegAudioFixPP())
 847             ydl.add_post_processor(AtomicParsleyPP())
 848
 849         # Update version
 850         if opts.update_self:
 851             update_self(ydl.to_screen, opts.verbose)
 852
 853         # Remove cache dir
 854         if opts.rm_cachedir:
 855             if opts.cachedir is None:
 856                 ydl.to_screen(u'No cache dir specified (Did you combine --no-cache-dir and --rm-cache-dir?)')
 857             else:
 858                 if ('.cache' not in opts.cachedir) or ('youtube-dl' not in opts.cachedir):
 859                     ydl.to_screen(u'Not removing directory %s - this does not look like a cache dir')
 860                     retcode = 141
 861                 else:
 862                     ydl.to_screen(
 863                         u'Removing cache dir %s .' % opts.cachedir,
 864                         skip_eol=True)
 865                     if os.path.exists(opts.cachedir):
 866                         ydl.to_screen(u'.', skip_eol=True)
 867                         shutil.rmtree(opts.cachedir)
 868                     ydl.to_screen(u'.')
 869
 870         # Maybe do nothing
 871         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 872             if not (opts.update_self or opts.rm_cachedir):
 873                 parser.error(u'you must provide at least one URL')
 874             else:
 875                 sys.exit()
 876
 877         try:
 878             if opts.load_info_filename is not None:
 879                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 880             else:
 881                 retcode = ydl.download(all_urls)
 882         except MaxDownloadsReached:
 883             ydl.to_screen(u'--max-download limit reached, aborting.')
 884             retcode = 101
 885
 886     sys.exit(retcode)
 887
 888
 889 def main(argv=None):
 890     try:
 891         _real_main(argv)
 892     except DownloadError:
 893         sys.exit(1)
 894     except SameFileError:
 895         sys.exit(u'ERROR: fixed output name but more than one file to download')
 896     except KeyboardInterrupt:
 897         sys.exit(u'\nERROR: Interrupted by user')