youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70     'David Fabijan',
  71     'Sebastian Haas',
  72     'Alexander Kirk',
  73     'Erik Johnson',
  74     'Keith Beckman',
  75     'Ole Ernst',
  76     'Aaron McDaniel (mcd1992)',
  77     'Magnus Kolstad',
  78 )
  79
  80 __license__ = 'Public Domain'
  81
  82 import codecs
  83 import io
  84 import optparse
  85 import os
  86 import random
  87 import shlex
  88 import sys
  89
  90
  91 from .utils import (
  92     compat_getpass,
  93     compat_print,
  94     DateRange,
  95     DEFAULT_OUTTMPL,
  96     decodeOption,
  97     get_term_width,
  98     DownloadError,
  99     MaxDownloadsReached,
 100     preferredencoding,
 101     read_batch_urls,
 102     SameFileError,
 103     setproctitle,
 104     std_headers,
 105     write_string,
 106 )
 107 from .update import update_self
 108 from .downloader import (
 109     FileDownloader,
 110 )
 111 from .extractor import gen_extractors
 112 from .version import __version__
 113 from .YoutubeDL import YoutubeDL
 114 from .postprocessor import (
 115     AtomicParsleyPP,
 116     FFmpegAudioFixPP,
 117     FFmpegMetadataPP,
 118     FFmpegVideoConvertor,
 119     FFmpegExtractAudioPP,
 120     FFmpegEmbedSubtitlePP,
 121     XAttrMetadataPP,
 122     ExecAfterDownloadPP,
 123 )
 124
 125
 126 def parseOpts(overrideArguments=None):
 127     def _readOptions(filename_bytes, default=[]):
 128         try:
 129             optionf = open(filename_bytes)
 130         except IOError:
 131             return default  # silently skip if file is not present
 132         try:
 133             res = []
 134             for l in optionf:
 135                 res += shlex.split(l, comments=True)
 136         finally:
 137             optionf.close()
 138         return res
 139
 140     def _readUserConf():
 141         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 142         if xdg_config_home:
 143             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 144             if not os.path.isfile(userConfFile):
 145                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 146         else:
 147             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 148             if not os.path.isfile(userConfFile):
 149                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 150         userConf = _readOptions(userConfFile, None)
 151
 152         if userConf is None:
 153             appdata_dir = os.environ.get('appdata')
 154             if appdata_dir:
 155                 userConf = _readOptions(
 156                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 157                     default=None)
 158                 if userConf is None:
 159                     userConf = _readOptions(
 160                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 161                         default=None)
 162
 163         if userConf is None:
 164             userConf = _readOptions(
 165                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 166                 default=None)
 167         if userConf is None:
 168             userConf = _readOptions(
 169                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 170                 default=None)
 171
 172         if userConf is None:
 173             userConf = []
 174
 175         return userConf
 176
 177     def _format_option_string(option):
 178         ''' ('-o', '--option') -> -o, --format METAVAR'''
 179
 180         opts = []
 181
 182         if option._short_opts:
 183             opts.append(option._short_opts[0])
 184         if option._long_opts:
 185             opts.append(option._long_opts[0])
 186         if len(opts) > 1:
 187             opts.insert(1, ', ')
 188
 189         if option.takes_value(): opts.append(' %s' % option.metavar)
 190
 191         return "".join(opts)
 192
 193     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 194         setattr(parser.values, option.dest, value.split(','))
 195
 196     def _hide_login_info(opts):
 197         opts = list(opts)
 198         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 199             try:
 200                 i = opts.index(private_opt)
 201                 opts[i+1] = '<PRIVATE>'
 202             except ValueError:
 203                 pass
 204         return opts
 205
 206     max_width = 80
 207     max_help_position = 80
 208
 209     # No need to wrap help messages if we're on a wide console
 210     columns = get_term_width()
 211     if columns: max_width = columns
 212
 213     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 214     fmt.format_option_strings = _format_option_string
 215
 216     kw = {
 217         'version'   : __version__,
 218         'formatter' : fmt,
 219         'usage' : '%prog [options] url [url...]',
 220         'conflict_handler' : 'resolve',
 221     }
 222
 223     parser = optparse.OptionParser(**kw)
 224
 225     # option groups
 226     general        = optparse.OptionGroup(parser, 'General Options')
 227     selection      = optparse.OptionGroup(parser, 'Video Selection')
 228     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 229     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 230     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 231     downloader     = optparse.OptionGroup(parser, 'Download Options')
 232     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 233     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 234     workarounds    = optparse.OptionGroup(parser, 'Workarounds')
 235     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 236
 237     general.add_option('-h', '--help',
 238             action='help', help='print this help text and exit')
 239     general.add_option('-v', '--version',
 240             action='version', help='print program version and exit')
 241     general.add_option('-U', '--update',
 242             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 243     general.add_option('-i', '--ignore-errors',
 244             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 245     general.add_option('--abort-on-error',
 246             action='store_false', dest='ignoreerrors',
 247             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 248     general.add_option('--dump-user-agent',
 249             action='store_true', dest='dump_user_agent',
 250             help='display the current browser identification', default=False)
 251     general.add_option('--list-extractors',
 252             action='store_true', dest='list_extractors',
 253             help='List all supported extractors and the URLs they would handle', default=False)
 254     general.add_option('--extractor-descriptions',
 255             action='store_true', dest='list_extractor_descriptions',
 256             help='Output descriptions of all supported extractors', default=False)
 257     general.add_option(
 258         '--proxy', dest='proxy', default=None, metavar='URL',
 259         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 260     general.add_option(
 261         '--socket-timeout', dest='socket_timeout',
 262         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 263     general.add_option(
 264         '--default-search',
 265         dest='default_search', metavar='PREFIX',
 266         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
 267     general.add_option(
 268         '--ignore-config',
 269         action='store_true',
 270         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 271
 272     selection.add_option(
 273         '--playlist-start',
 274         dest='playliststart', metavar='NUMBER', default=1, type=int,
 275         help='playlist video to start at (default is %default)')
 276     selection.add_option(
 277         '--playlist-end',
 278         dest='playlistend', metavar='NUMBER', default=None, type=int,
 279         help='playlist video to end at (default is last)')
 280     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 281     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 282     selection.add_option('--max-downloads', metavar='NUMBER',
 283                          dest='max_downloads', type=int, default=None,
 284                          help='Abort after downloading NUMBER files')
 285     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 286     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 287     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 288     selection.add_option(
 289         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 290         help='download only videos uploaded on or before this date (i.e. inclusive)')
 291     selection.add_option(
 292         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 293         help='download only videos uploaded on or after this date (i.e. inclusive)')
 294     selection.add_option(
 295         '--min-views', metavar='COUNT', dest='min_views',
 296         default=None, type=int,
 297         help="Do not download any videos with less than COUNT views",)
 298     selection.add_option(
 299         '--max-views', metavar='COUNT', dest='max_views',
 300         default=None, type=int,
 301         help="Do not download any videos with more than COUNT views",)
 302     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 303     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 304                          help='download only videos suitable for the given age',
 305                          default=None, type=int)
 306     selection.add_option('--download-archive', metavar='FILE',
 307                          dest='download_archive',
 308                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 309     selection.add_option(
 310         '--include-ads', dest='include_ads',
 311         action='store_true',
 312         help='Download advertisements as well (experimental)')
 313     selection.add_option(
 314         '--youtube-include-dash-manifest', action='store_true',
 315         dest='youtube_include_dash_manifest', default=False,
 316         help='Try to download the DASH manifest on YouTube videos (experimental)')
 317
 318     authentication.add_option('-u', '--username',
 319             dest='username', metavar='USERNAME', help='account username')
 320     authentication.add_option('-p', '--password',
 321             dest='password', metavar='PASSWORD', help='account password')
 322     authentication.add_option('-2', '--twofactor',
 323             dest='twofactor', metavar='TWOFACTOR', help='two-factor auth code')
 324     authentication.add_option('-n', '--netrc',
 325             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 326     authentication.add_option('--video-password',
 327             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 328
 329
 330     video_format.add_option('-f', '--format',
 331             action='store', dest='format', metavar='FORMAT', default=None,
 332             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 333     video_format.add_option('--all-formats',
 334             action='store_const', dest='format', help='download all available video formats', const='all')
 335     video_format.add_option('--prefer-free-formats',
 336             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 337     video_format.add_option('--max-quality',
 338             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 339     video_format.add_option('-F', '--list-formats',
 340             action='store_true', dest='listformats', help='list all available formats')
 341
 342     subtitles.add_option('--write-sub', '--write-srt',
 343             action='store_true', dest='writesubtitles',
 344             help='write subtitle file', default=False)
 345     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 346             action='store_true', dest='writeautomaticsub',
 347             help='write automatic subtitle file (youtube only)', default=False)
 348     subtitles.add_option('--all-subs',
 349             action='store_true', dest='allsubtitles',
 350             help='downloads all the available subtitles of the video', default=False)
 351     subtitles.add_option('--list-subs',
 352             action='store_true', dest='listsubtitles',
 353             help='lists all available subtitles for the video', default=False)
 354     subtitles.add_option('--sub-format',
 355             action='store', dest='subtitlesformat', metavar='FORMAT',
 356             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 357     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 358             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 359             default=[], callback=_comma_separated_values_options_callback,
 360             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 361
 362     downloader.add_option('-r', '--rate-limit',
 363             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 364     downloader.add_option('-R', '--retries',
 365             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 366     downloader.add_option('--buffer-size',
 367             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 368     downloader.add_option('--no-resize-buffer',
 369             action='store_true', dest='noresizebuffer',
 370             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 371     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 372
 373     workarounds.add_option(
 374         '--encoding', dest='encoding', metavar='ENCODING',
 375         help='Force the specified encoding (experimental)')
 376     workarounds.add_option(
 377         '--no-check-certificate', action='store_true',
 378         dest='no_check_certificate', default=False,
 379         help='Suppress HTTPS certificate validation.')
 380     workarounds.add_option(
 381         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 382         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 383     workarounds.add_option(
 384         '--user-agent', metavar='UA',
 385         dest='user_agent', help='specify a custom user agent')
 386     workarounds.add_option(
 387         '--referer', metavar='REF',
 388         dest='referer', default=None,
 389         help='specify a custom referer, use if the video access is restricted to one domain',
 390     )
 391     workarounds.add_option(
 392         '--add-header', metavar='FIELD:VALUE',
 393         dest='headers', action='append',
 394         help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
 395     )
 396     workarounds.add_option(
 397         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 398         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 399
 400     verbosity.add_option('-q', '--quiet',
 401             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 402     verbosity.add_option(
 403         '--no-warnings',
 404         dest='no_warnings', action='store_true', default=False,
 405         help='Ignore warnings')
 406     verbosity.add_option('-s', '--simulate',
 407             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 408     verbosity.add_option('--skip-download',
 409             action='store_true', dest='skip_download', help='do not download the video', default=False)
 410     verbosity.add_option('-g', '--get-url',
 411             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 412     verbosity.add_option('-e', '--get-title',
 413             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 414     verbosity.add_option('--get-id',
 415             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 416     verbosity.add_option('--get-thumbnail',
 417             action='store_true', dest='getthumbnail',
 418             help='simulate, quiet but print thumbnail URL', default=False)
 419     verbosity.add_option('--get-description',
 420             action='store_true', dest='getdescription',
 421             help='simulate, quiet but print video description', default=False)
 422     verbosity.add_option('--get-duration',
 423             action='store_true', dest='getduration',
 424             help='simulate, quiet but print video length', default=False)
 425     verbosity.add_option('--get-filename',
 426             action='store_true', dest='getfilename',
 427             help='simulate, quiet but print output filename', default=False)
 428     verbosity.add_option('--get-format',
 429             action='store_true', dest='getformat',
 430             help='simulate, quiet but print output format', default=False)
 431     verbosity.add_option('-j', '--dump-json',
 432             action='store_true', dest='dumpjson',
 433             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 434     verbosity.add_option('--newline',
 435             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 436     verbosity.add_option('--no-progress',
 437             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 438     verbosity.add_option('--console-title',
 439             action='store_true', dest='consoletitle',
 440             help='display progress in console titlebar', default=False)
 441     verbosity.add_option('-v', '--verbose',
 442             action='store_true', dest='verbose', help='print various debugging information', default=False)
 443     verbosity.add_option('--dump-intermediate-pages',
 444             action='store_true', dest='dump_intermediate_pages', default=False,
 445             help='print downloaded pages to debug problems (very verbose)')
 446     verbosity.add_option('--write-pages',
 447             action='store_true', dest='write_pages', default=False,
 448             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 449     verbosity.add_option('--youtube-print-sig-code',
 450             action='store_true', dest='youtube_print_sig_code', default=False,
 451             help=optparse.SUPPRESS_HELP)
 452     verbosity.add_option('--print-traffic',
 453             dest='debug_printtraffic', action='store_true', default=False,
 454             help='Display sent and read HTTP traffic')
 455
 456
 457     filesystem.add_option('-a', '--batch-file',
 458             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 459     filesystem.add_option('--id',
 460             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 461     filesystem.add_option('-A', '--auto-number',
 462             action='store_true', dest='autonumber',
 463             help='number downloaded files starting from 00000', default=False)
 464     filesystem.add_option('-o', '--output',
 465             dest='outtmpl', metavar='TEMPLATE',
 466             help=('output filename template. Use %(title)s to get the title, '
 467                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 468                   '%(autonumber)s to get an automatically incremented number, '
 469                   '%(ext)s for the filename extension, '
 470                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 471                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 472                   '%(upload_date)s for the upload date (YYYYMMDD), '
 473                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 474                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 475                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 476                   '%(height)s and %(width)s for the width and height of the video format. '
 477                   '%(resolution)s for a textual description of the resolution of the video format. '
 478                   'Use - to output to stdout. Can also be used to download to a different directory, '
 479                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 480     filesystem.add_option('--autonumber-size',
 481             dest='autonumber_size', metavar='NUMBER',
 482             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 483     filesystem.add_option('--restrict-filenames',
 484             action='store_true', dest='restrictfilenames',
 485             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 486     filesystem.add_option('-t', '--title',
 487             action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
 488     filesystem.add_option('-l', '--literal',
 489             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 490     filesystem.add_option('-w', '--no-overwrites',
 491             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 492     filesystem.add_option('-c', '--continue',
 493             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 494     filesystem.add_option('--no-continue',
 495             action='store_false', dest='continue_dl',
 496             help='do not resume partially downloaded files (restart from beginning)')
 497     filesystem.add_option('--no-part',
 498             action='store_true', dest='nopart', help='do not use .part files', default=False)
 499     filesystem.add_option('--no-mtime',
 500             action='store_false', dest='updatetime',
 501             help='do not use the Last-modified header to set the file modification time', default=True)
 502     filesystem.add_option('--write-description',
 503             action='store_true', dest='writedescription',
 504             help='write video description to a .description file', default=False)
 505     filesystem.add_option('--write-info-json',
 506             action='store_true', dest='writeinfojson',
 507             help='write video metadata to a .info.json file', default=False)
 508     filesystem.add_option('--write-annotations',
 509             action='store_true', dest='writeannotations',
 510             help='write video annotations to a .annotation file', default=False)
 511     filesystem.add_option('--write-thumbnail',
 512             action='store_true', dest='writethumbnail',
 513             help='write thumbnail image to disk', default=False)
 514     filesystem.add_option('--load-info',
 515             dest='load_info_filename', metavar='FILE',
 516             help='json file containing the video information (created with the "--write-json" option)')
 517     filesystem.add_option('--cookies',
 518             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 519     filesystem.add_option(
 520         '--cache-dir', dest='cachedir', default=None, metavar='DIR',
 521         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 522     filesystem.add_option(
 523         '--no-cache-dir', action='store_const', const=False, dest='cachedir',
 524         help='Disable filesystem caching')
 525     filesystem.add_option(
 526         '--rm-cache-dir', action='store_true', dest='rm_cachedir',
 527         help='Delete all filesystem cache files')
 528
 529
 530     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 531             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 532     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 533             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 534     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 535             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 536     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 537             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
 538     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 539             help='keeps the video file on disk after the post-processing; the video is erased by default')
 540     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 541             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 542     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 543             help='embed subtitles in the video (only for mp4 videos)')
 544     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 545             help='embed thumbnail in the audio as cover art')
 546     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 547             help='write metadata to the video file')
 548     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 549             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 550     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 551         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 552     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 553         help='Prefer ffmpeg over avconv for running the postprocessors')
 554     postproc.add_option(
 555         '--exec', metavar='CMD', dest='exec_cmd',
 556         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'' )
 557
 558     parser.add_option_group(general)
 559     parser.add_option_group(selection)
 560     parser.add_option_group(downloader)
 561     parser.add_option_group(filesystem)
 562     parser.add_option_group(verbosity)
 563     parser.add_option_group(workarounds)
 564     parser.add_option_group(video_format)
 565     parser.add_option_group(subtitles)
 566     parser.add_option_group(authentication)
 567     parser.add_option_group(postproc)
 568
 569     if overrideArguments is not None:
 570         opts, args = parser.parse_args(overrideArguments)
 571         if opts.verbose:
 572             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 573     else:
 574         commandLineConf = sys.argv[1:]
 575         if '--ignore-config' in commandLineConf:
 576             systemConf = []
 577             userConf = []
 578         else:
 579             systemConf = _readOptions('/etc/youtube-dl.conf')
 580             if '--ignore-config' in systemConf:
 581                 userConf = []
 582             else:
 583                 userConf = _readUserConf()
 584         argv = systemConf + userConf + commandLineConf
 585
 586         opts, args = parser.parse_args(argv)
 587         if opts.verbose:
 588             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 589             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 590             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 591
 592     return parser, opts, args
 593
 594
 595 def _real_main(argv=None):
 596     # Compatibility fixes for Windows
 597     if sys.platform == 'win32':
 598         # https://github.com/rg3/youtube-dl/issues/820
 599         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 600
 601     setproctitle(u'youtube-dl')
 602
 603     parser, opts, args = parseOpts(argv)
 604
 605     # Set user agent
 606     if opts.user_agent is not None:
 607         std_headers['User-Agent'] = opts.user_agent
 608
 609     # Set referer
 610     if opts.referer is not None:
 611         std_headers['Referer'] = opts.referer
 612
 613     # Custom HTTP headers
 614     if opts.headers is not None:
 615         for h in opts.headers:
 616             if h.find(':', 1) < 0:
 617                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 618             key, value = h.split(':', 2)
 619             if opts.verbose:
 620                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 621             std_headers[key] = value
 622
 623     # Dump user agent
 624     if opts.dump_user_agent:
 625         compat_print(std_headers['User-Agent'])
 626         sys.exit(0)
 627
 628     # Batch file verification
 629     batch_urls = []
 630     if opts.batchfile is not None:
 631         try:
 632             if opts.batchfile == '-':
 633                 batchfd = sys.stdin
 634             else:
 635                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 636             batch_urls = read_batch_urls(batchfd)
 637             if opts.verbose:
 638                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 639         except IOError:
 640             sys.exit(u'ERROR: batch file could not be read')
 641     all_urls = batch_urls + args
 642     all_urls = [url.strip() for url in all_urls]
 643     _enc = preferredencoding()
 644     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 645
 646     extractors = gen_extractors()
 647
 648     if opts.list_extractors:
 649         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 650             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 651             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 652             for mu in matchedUrls:
 653                 compat_print(u'  ' + mu)
 654         sys.exit(0)
 655     if opts.list_extractor_descriptions:
 656         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 657             if not ie._WORKING:
 658                 continue
 659             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 660             if desc is False:
 661                 continue
 662             if hasattr(ie, 'SEARCH_KEY'):
 663                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 664                 _COUNTS = (u'', u'5', u'10', u'all')
 665                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 666             compat_print(desc)
 667         sys.exit(0)
 668
 669
 670     # Conflicting, missing and erroneous options
 671     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 672         parser.error(u'using .netrc conflicts with giving username/password')
 673     if opts.password is not None and opts.username is None:
 674         parser.error(u'account username missing\n')
 675     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 676         parser.error(u'using output template conflicts with using title, video ID or auto number')
 677     if opts.usetitle and opts.useid:
 678         parser.error(u'using title conflicts with using video ID')
 679     if opts.username is not None and opts.password is None:
 680         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 681     if opts.ratelimit is not None:
 682         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 683         if numeric_limit is None:
 684             parser.error(u'invalid rate limit specified')
 685         opts.ratelimit = numeric_limit
 686     if opts.min_filesize is not None:
 687         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 688         if numeric_limit is None:
 689             parser.error(u'invalid min_filesize specified')
 690         opts.min_filesize = numeric_limit
 691     if opts.max_filesize is not None:
 692         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 693         if numeric_limit is None:
 694             parser.error(u'invalid max_filesize specified')
 695         opts.max_filesize = numeric_limit
 696     if opts.retries is not None:
 697         try:
 698             opts.retries = int(opts.retries)
 699         except (TypeError, ValueError):
 700             parser.error(u'invalid retry count specified')
 701     if opts.buffersize is not None:
 702         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 703         if numeric_buffersize is None:
 704             parser.error(u'invalid buffer size specified')
 705         opts.buffersize = numeric_buffersize
 706     if opts.playliststart <= 0:
 707         raise ValueError(u'Playlist start must be positive')
 708     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 709         raise ValueError(u'Playlist end must be greater than playlist start')
 710     if opts.extractaudio:
 711         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 712             parser.error(u'invalid audio format specified')
 713     if opts.audioquality:
 714         opts.audioquality = opts.audioquality.strip('k').strip('K')
 715         if not opts.audioquality.isdigit():
 716             parser.error(u'invalid audio quality specified')
 717     if opts.recodevideo is not None:
 718         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 719             parser.error(u'invalid video recode format specified')
 720     if opts.date is not None:
 721         date = DateRange.day(opts.date)
 722     else:
 723         date = DateRange(opts.dateafter, opts.datebefore)
 724     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 725         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 726
 727     # Do not download videos when there are audio-only formats
 728     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 729         opts.format = 'bestaudio/best'
 730
 731     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 732     # this was the old behaviour if only --all-sub was given.
 733     if opts.allsubtitles and (opts.writeautomaticsub == False):
 734         opts.writesubtitles = True
 735
 736     if sys.version_info < (3,):
 737         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 738         if opts.outtmpl is not None:
 739             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 740     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 741             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 742             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 743             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 744             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 745             or (opts.useid and u'%(id)s.%(ext)s')
 746             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 747             or DEFAULT_OUTTMPL)
 748     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 749         parser.error(u'Cannot download a video and extract audio into the same'
 750                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 751                      u' template'.format(outtmpl))
 752
 753     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 754     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 755
 756     ydl_opts = {
 757         'usenetrc': opts.usenetrc,
 758         'username': opts.username,
 759         'password': opts.password,
 760         'twofactor': opts.twofactor,
 761         'videopassword': opts.videopassword,
 762         'quiet': (opts.quiet or any_printing),
 763         'no_warnings': opts.no_warnings,
 764         'forceurl': opts.geturl,
 765         'forcetitle': opts.gettitle,
 766         'forceid': opts.getid,
 767         'forcethumbnail': opts.getthumbnail,
 768         'forcedescription': opts.getdescription,
 769         'forceduration': opts.getduration,
 770         'forcefilename': opts.getfilename,
 771         'forceformat': opts.getformat,
 772         'forcejson': opts.dumpjson,
 773         'simulate': opts.simulate,
 774         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 775         'format': opts.format,
 776         'format_limit': opts.format_limit,
 777         'listformats': opts.listformats,
 778         'outtmpl': outtmpl,
 779         'autonumber_size': opts.autonumber_size,
 780         'restrictfilenames': opts.restrictfilenames,
 781         'ignoreerrors': opts.ignoreerrors,
 782         'ratelimit': opts.ratelimit,
 783         'nooverwrites': opts.nooverwrites,
 784         'retries': opts.retries,
 785         'buffersize': opts.buffersize,
 786         'noresizebuffer': opts.noresizebuffer,
 787         'continuedl': opts.continue_dl,
 788         'noprogress': opts.noprogress,
 789         'progress_with_newline': opts.progress_with_newline,
 790         'playliststart': opts.playliststart,
 791         'playlistend': opts.playlistend,
 792         'noplaylist': opts.noplaylist,
 793         'logtostderr': opts.outtmpl == '-',
 794         'consoletitle': opts.consoletitle,
 795         'nopart': opts.nopart,
 796         'updatetime': opts.updatetime,
 797         'writedescription': opts.writedescription,
 798         'writeannotations': opts.writeannotations,
 799         'writeinfojson': opts.writeinfojson,
 800         'writethumbnail': opts.writethumbnail,
 801         'writesubtitles': opts.writesubtitles,
 802         'writeautomaticsub': opts.writeautomaticsub,
 803         'allsubtitles': opts.allsubtitles,
 804         'listsubtitles': opts.listsubtitles,
 805         'subtitlesformat': opts.subtitlesformat,
 806         'subtitleslangs': opts.subtitleslangs,
 807         'matchtitle': decodeOption(opts.matchtitle),
 808         'rejecttitle': decodeOption(opts.rejecttitle),
 809         'max_downloads': opts.max_downloads,
 810         'prefer_free_formats': opts.prefer_free_formats,
 811         'verbose': opts.verbose,
 812         'dump_intermediate_pages': opts.dump_intermediate_pages,
 813         'write_pages': opts.write_pages,
 814         'test': opts.test,
 815         'keepvideo': opts.keepvideo,
 816         'min_filesize': opts.min_filesize,
 817         'max_filesize': opts.max_filesize,
 818         'min_views': opts.min_views,
 819         'max_views': opts.max_views,
 820         'daterange': date,
 821         'cachedir': opts.cachedir,
 822         'youtube_print_sig_code': opts.youtube_print_sig_code,
 823         'age_limit': opts.age_limit,
 824         'download_archive': download_archive_fn,
 825         'cookiefile': opts.cookiefile,
 826         'nocheckcertificate': opts.no_check_certificate,
 827         'prefer_insecure': opts.prefer_insecure,
 828         'proxy': opts.proxy,
 829         'socket_timeout': opts.socket_timeout,
 830         'bidi_workaround': opts.bidi_workaround,
 831         'debug_printtraffic': opts.debug_printtraffic,
 832         'prefer_ffmpeg': opts.prefer_ffmpeg,
 833         'include_ads': opts.include_ads,
 834         'default_search': opts.default_search,
 835         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 836         'encoding': opts.encoding,
 837         'exec_cmd': opts.exec_cmd,
 838     }
 839
 840     with YoutubeDL(ydl_opts) as ydl:
 841         ydl.print_debug_header()
 842         ydl.add_default_info_extractors()
 843
 844         # PostProcessors
 845         # Add the metadata pp first, the other pps will copy it
 846         if opts.addmetadata:
 847             ydl.add_post_processor(FFmpegMetadataPP())
 848         if opts.extractaudio:
 849             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 850         if opts.recodevideo:
 851             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 852         if opts.embedsubtitles:
 853             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 854         if opts.xattrs:
 855             ydl.add_post_processor(XAttrMetadataPP())
 856         if opts.embedthumbnail:
 857             if not opts.addmetadata:
 858                 ydl.add_post_processor(FFmpegAudioFixPP())
 859             ydl.add_post_processor(AtomicParsleyPP())
 860
 861
 862         # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
 863         # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
 864         if opts.exec_cmd:
 865             ydl.add_post_processor(ExecAfterDownloadPP(
 866                 verboseOutput=opts.verbose, exec_cmd=opts.exec_cmd))
 867
 868         # Update version
 869         if opts.update_self:
 870             update_self(ydl.to_screen, opts.verbose)
 871
 872         # Remove cache dir
 873         if opts.rm_cachedir:
 874             ydl.cache.remove()
 875
 876         # Maybe do nothing
 877         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 878             if not (opts.update_self or opts.rm_cachedir):
 879                 parser.error(u'you must provide at least one URL')
 880             else:
 881                 sys.exit()
 882
 883         try:
 884             if opts.load_info_filename is not None:
 885                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 886             else:
 887                 retcode = ydl.download(all_urls)
 888         except MaxDownloadsReached:
 889             ydl.to_screen(u'--max-download limit reached, aborting.')
 890             retcode = 101
 891
 892     sys.exit(retcode)
 893
 894
 895 def main(argv=None):
 896     try:
 897         _real_main(argv)
 898     except DownloadError:
 899         sys.exit(1)
 900     except SameFileError:
 901         sys.exit(u'ERROR: fixed output name but more than one file to download')
 902     except KeyboardInterrupt:
 903         sys.exit(u'\nERROR: Interrupted by user')