youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70     'David Fabijan',
  71     'Sebastian Haas',
  72     'Alexander Kirk',
  73     'Erik Johnson',
  74     'Keith Beckman',
  75     'Ole Ernst',
  76     'Aaron McDaniel (mcd1992)',
  77 )
  78
  79 __license__ = 'Public Domain'
  80
  81 import codecs
  82 import io
  83 import optparse
  84 import os
  85 import random
  86 import shlex
  87 import shutil
  88 import sys
  89
  90
  91 from .utils import (
  92     compat_getpass,
  93     compat_print,
  94     DateRange,
  95     DEFAULT_OUTTMPL,
  96     decodeOption,
  97     get_term_width,
  98     DownloadError,
  99     get_cachedir,
 100     MaxDownloadsReached,
 101     preferredencoding,
 102     read_batch_urls,
 103     SameFileError,
 104     setproctitle,
 105     std_headers,
 106     write_string,
 107 )
 108 from .update import update_self
 109 from .downloader import (
 110     FileDownloader,
 111 )
 112 from .extractor import gen_extractors
 113 from .version import __version__
 114 from .YoutubeDL import YoutubeDL
 115 from .postprocessor import (
 116     AtomicParsleyPP,
 117     FFmpegAudioFixPP,
 118     FFmpegMetadataPP,
 119     FFmpegVideoConvertor,
 120     FFmpegExtractAudioPP,
 121     FFmpegEmbedSubtitlePP,
 122     XAttrMetadataPP,
 123     ExecAfterDownloadPP,
 124 )
 125
 126
 127 def parseOpts(overrideArguments=None):
 128     def _readOptions(filename_bytes, default=[]):
 129         try:
 130             optionf = open(filename_bytes)
 131         except IOError:
 132             return default  # silently skip if file is not present
 133         try:
 134             res = []
 135             for l in optionf:
 136                 res += shlex.split(l, comments=True)
 137         finally:
 138             optionf.close()
 139         return res
 140
 141     def _readUserConf():
 142         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 143         if xdg_config_home:
 144             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 145             if not os.path.isfile(userConfFile):
 146                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 147         else:
 148             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 149             if not os.path.isfile(userConfFile):
 150                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 151         userConf = _readOptions(userConfFile, None)
 152
 153         if userConf is None:
 154             appdata_dir = os.environ.get('appdata')
 155             if appdata_dir:
 156                 userConf = _readOptions(
 157                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 158                     default=None)
 159                 if userConf is None:
 160                     userConf = _readOptions(
 161                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 162                         default=None)
 163
 164         if userConf is None:
 165             userConf = _readOptions(
 166                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 167                 default=None)
 168         if userConf is None:
 169             userConf = _readOptions(
 170                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 171                 default=None)
 172
 173         if userConf is None:
 174             userConf = []
 175
 176         return userConf
 177
 178     def _format_option_string(option):
 179         ''' ('-o', '--option') -> -o, --format METAVAR'''
 180
 181         opts = []
 182
 183         if option._short_opts:
 184             opts.append(option._short_opts[0])
 185         if option._long_opts:
 186             opts.append(option._long_opts[0])
 187         if len(opts) > 1:
 188             opts.insert(1, ', ')
 189
 190         if option.takes_value(): opts.append(' %s' % option.metavar)
 191
 192         return "".join(opts)
 193
 194     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 195         setattr(parser.values, option.dest, value.split(','))
 196
 197     def _hide_login_info(opts):
 198         opts = list(opts)
 199         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 200             try:
 201                 i = opts.index(private_opt)
 202                 opts[i+1] = '<PRIVATE>'
 203             except ValueError:
 204                 pass
 205         return opts
 206
 207     max_width = 80
 208     max_help_position = 80
 209
 210     # No need to wrap help messages if we're on a wide console
 211     columns = get_term_width()
 212     if columns: max_width = columns
 213
 214     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 215     fmt.format_option_strings = _format_option_string
 216
 217     kw = {
 218         'version'   : __version__,
 219         'formatter' : fmt,
 220         'usage' : '%prog [options] url [url...]',
 221         'conflict_handler' : 'resolve',
 222     }
 223
 224     parser = optparse.OptionParser(**kw)
 225
 226     # option groups
 227     general        = optparse.OptionGroup(parser, 'General Options')
 228     selection      = optparse.OptionGroup(parser, 'Video Selection')
 229     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 230     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 231     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 232     downloader     = optparse.OptionGroup(parser, 'Download Options')
 233     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 234     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 235     workarounds    = optparse.OptionGroup(parser, 'Workarounds')
 236     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 237
 238     general.add_option('-h', '--help',
 239             action='help', help='print this help text and exit')
 240     general.add_option('-v', '--version',
 241             action='version', help='print program version and exit')
 242     general.add_option('-U', '--update',
 243             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 244     general.add_option('-i', '--ignore-errors',
 245             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 246     general.add_option('--abort-on-error',
 247             action='store_false', dest='ignoreerrors',
 248             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 249     general.add_option('--dump-user-agent',
 250             action='store_true', dest='dump_user_agent',
 251             help='display the current browser identification', default=False)
 252     general.add_option('--list-extractors',
 253             action='store_true', dest='list_extractors',
 254             help='List all supported extractors and the URLs they would handle', default=False)
 255     general.add_option('--extractor-descriptions',
 256             action='store_true', dest='list_extractor_descriptions',
 257             help='Output descriptions of all supported extractors', default=False)
 258     general.add_option(
 259         '--proxy', dest='proxy', default=None, metavar='URL',
 260         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 261     general.add_option(
 262         '--socket-timeout', dest='socket_timeout',
 263         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 264     general.add_option(
 265         '--default-search',
 266         dest='default_search', metavar='PREFIX',
 267         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
 268     general.add_option(
 269         '--ignore-config',
 270         action='store_true',
 271         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 272
 273     selection.add_option(
 274         '--playlist-start',
 275         dest='playliststart', metavar='NUMBER', default=1, type=int,
 276         help='playlist video to start at (default is %default)')
 277     selection.add_option(
 278         '--playlist-end',
 279         dest='playlistend', metavar='NUMBER', default=None, type=int,
 280         help='playlist video to end at (default is last)')
 281     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 282     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 283     selection.add_option('--max-downloads', metavar='NUMBER',
 284                          dest='max_downloads', type=int, default=None,
 285                          help='Abort after downloading NUMBER files')
 286     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 287     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 288     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 289     selection.add_option(
 290         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 291         help='download only videos uploaded on or before this date (i.e. inclusive)')
 292     selection.add_option(
 293         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 294         help='download only videos uploaded on or after this date (i.e. inclusive)')
 295     selection.add_option(
 296         '--min-views', metavar='COUNT', dest='min_views',
 297         default=None, type=int,
 298         help="Do not download any videos with less than COUNT views",)
 299     selection.add_option(
 300         '--max-views', metavar='COUNT', dest='max_views',
 301         default=None, type=int,
 302         help="Do not download any videos with more than COUNT views",)
 303     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 304     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 305                          help='download only videos suitable for the given age',
 306                          default=None, type=int)
 307     selection.add_option('--download-archive', metavar='FILE',
 308                          dest='download_archive',
 309                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 310     selection.add_option(
 311         '--include-ads', dest='include_ads',
 312         action='store_true',
 313         help='Download advertisements as well (experimental)')
 314     selection.add_option(
 315         '--youtube-include-dash-manifest', action='store_true',
 316         dest='youtube_include_dash_manifest', default=False,
 317         help='Try to download the DASH manifest on YouTube videos (experimental)')
 318
 319     authentication.add_option('-u', '--username',
 320             dest='username', metavar='USERNAME', help='account username')
 321     authentication.add_option('-p', '--password',
 322             dest='password', metavar='PASSWORD', help='account password')
 323     authentication.add_option('-2', '--twofactor',
 324             dest='twofactor', metavar='TWOFACTOR', help='two-factor auth code')
 325     authentication.add_option('-n', '--netrc',
 326             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 327     authentication.add_option('--video-password',
 328             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 329
 330
 331     video_format.add_option('-f', '--format',
 332             action='store', dest='format', metavar='FORMAT', default=None,
 333             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 334     video_format.add_option('--all-formats',
 335             action='store_const', dest='format', help='download all available video formats', const='all')
 336     video_format.add_option('--prefer-free-formats',
 337             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 338     video_format.add_option('--max-quality',
 339             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 340     video_format.add_option('-F', '--list-formats',
 341             action='store_true', dest='listformats', help='list all available formats')
 342
 343     subtitles.add_option('--write-sub', '--write-srt',
 344             action='store_true', dest='writesubtitles',
 345             help='write subtitle file', default=False)
 346     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 347             action='store_true', dest='writeautomaticsub',
 348             help='write automatic subtitle file (youtube only)', default=False)
 349     subtitles.add_option('--all-subs',
 350             action='store_true', dest='allsubtitles',
 351             help='downloads all the available subtitles of the video', default=False)
 352     subtitles.add_option('--list-subs',
 353             action='store_true', dest='listsubtitles',
 354             help='lists all available subtitles for the video', default=False)
 355     subtitles.add_option('--sub-format',
 356             action='store', dest='subtitlesformat', metavar='FORMAT',
 357             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 358     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 359             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 360             default=[], callback=_comma_separated_values_options_callback,
 361             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 362
 363     downloader.add_option('-r', '--rate-limit',
 364             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 365     downloader.add_option('-R', '--retries',
 366             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 367     downloader.add_option('--buffer-size',
 368             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 369     downloader.add_option('--no-resize-buffer',
 370             action='store_true', dest='noresizebuffer',
 371             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 372     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 373
 374     workarounds.add_option(
 375         '--encoding', dest='encoding', metavar='ENCODING',
 376         help='Force the specified encoding (experimental)')
 377     workarounds.add_option(
 378         '--no-check-certificate', action='store_true',
 379         dest='no_check_certificate', default=False,
 380         help='Suppress HTTPS certificate validation.')
 381     workarounds.add_option(
 382         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 383         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 384     workarounds.add_option(
 385         '--user-agent', metavar='UA',
 386         dest='user_agent', help='specify a custom user agent')
 387     workarounds.add_option(
 388         '--referer', metavar='REF',
 389         dest='referer', default=None,
 390         help='specify a custom referer, use if the video access is restricted to one domain',
 391     )
 392     workarounds.add_option(
 393         '--add-header', metavar='FIELD:VALUE',
 394         dest='headers', action='append',
 395         help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
 396     )
 397     workarounds.add_option(
 398         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 399         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 400
 401     verbosity.add_option('-q', '--quiet',
 402             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 403     verbosity.add_option(
 404         '--no-warnings',
 405         dest='no_warnings', action='store_true', default=False,
 406         help='Ignore warnings')
 407     verbosity.add_option('-s', '--simulate',
 408             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 409     verbosity.add_option('--skip-download',
 410             action='store_true', dest='skip_download', help='do not download the video', default=False)
 411     verbosity.add_option('-g', '--get-url',
 412             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 413     verbosity.add_option('-e', '--get-title',
 414             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 415     verbosity.add_option('--get-id',
 416             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 417     verbosity.add_option('--get-thumbnail',
 418             action='store_true', dest='getthumbnail',
 419             help='simulate, quiet but print thumbnail URL', default=False)
 420     verbosity.add_option('--get-description',
 421             action='store_true', dest='getdescription',
 422             help='simulate, quiet but print video description', default=False)
 423     verbosity.add_option('--get-duration',
 424             action='store_true', dest='getduration',
 425             help='simulate, quiet but print video length', default=False)
 426     verbosity.add_option('--get-filename',
 427             action='store_true', dest='getfilename',
 428             help='simulate, quiet but print output filename', default=False)
 429     verbosity.add_option('--get-format',
 430             action='store_true', dest='getformat',
 431             help='simulate, quiet but print output format', default=False)
 432     verbosity.add_option('-j', '--dump-json',
 433             action='store_true', dest='dumpjson',
 434             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 435     verbosity.add_option('--newline',
 436             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 437     verbosity.add_option('--no-progress',
 438             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 439     verbosity.add_option('--console-title',
 440             action='store_true', dest='consoletitle',
 441             help='display progress in console titlebar', default=False)
 442     verbosity.add_option('-v', '--verbose',
 443             action='store_true', dest='verbose', help='print various debugging information', default=False)
 444     verbosity.add_option('--dump-intermediate-pages',
 445             action='store_true', dest='dump_intermediate_pages', default=False,
 446             help='print downloaded pages to debug problems (very verbose)')
 447     verbosity.add_option('--write-pages',
 448             action='store_true', dest='write_pages', default=False,
 449             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 450     verbosity.add_option('--youtube-print-sig-code',
 451             action='store_true', dest='youtube_print_sig_code', default=False,
 452             help=optparse.SUPPRESS_HELP)
 453     verbosity.add_option('--print-traffic',
 454             dest='debug_printtraffic', action='store_true', default=False,
 455             help='Display sent and read HTTP traffic')
 456
 457
 458     filesystem.add_option('-a', '--batch-file',
 459             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 460     filesystem.add_option('--id',
 461             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 462     filesystem.add_option('-A', '--auto-number',
 463             action='store_true', dest='autonumber',
 464             help='number downloaded files starting from 00000', default=False)
 465     filesystem.add_option('-o', '--output',
 466             dest='outtmpl', metavar='TEMPLATE',
 467             help=('output filename template. Use %(title)s to get the title, '
 468                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 469                   '%(autonumber)s to get an automatically incremented number, '
 470                   '%(ext)s for the filename extension, '
 471                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 472                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 473                   '%(upload_date)s for the upload date (YYYYMMDD), '
 474                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 475                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 476                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 477                   '%(height)s and %(width)s for the width and height of the video format. '
 478                   '%(resolution)s for a textual description of the resolution of the video format. '
 479                   'Use - to output to stdout. Can also be used to download to a different directory, '
 480                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 481     filesystem.add_option('--autonumber-size',
 482             dest='autonumber_size', metavar='NUMBER',
 483             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 484     filesystem.add_option('--restrict-filenames',
 485             action='store_true', dest='restrictfilenames',
 486             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 487     filesystem.add_option('-t', '--title',
 488             action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
 489     filesystem.add_option('-l', '--literal',
 490             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 491     filesystem.add_option('-w', '--no-overwrites',
 492             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 493     filesystem.add_option('-c', '--continue',
 494             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 495     filesystem.add_option('--no-continue',
 496             action='store_false', dest='continue_dl',
 497             help='do not resume partially downloaded files (restart from beginning)')
 498     filesystem.add_option('--no-part',
 499             action='store_true', dest='nopart', help='do not use .part files', default=False)
 500     filesystem.add_option('--no-mtime',
 501             action='store_false', dest='updatetime',
 502             help='do not use the Last-modified header to set the file modification time', default=True)
 503     filesystem.add_option('--write-description',
 504             action='store_true', dest='writedescription',
 505             help='write video description to a .description file', default=False)
 506     filesystem.add_option('--write-info-json',
 507             action='store_true', dest='writeinfojson',
 508             help='write video metadata to a .info.json file', default=False)
 509     filesystem.add_option('--write-annotations',
 510             action='store_true', dest='writeannotations',
 511             help='write video annotations to a .annotation file', default=False)
 512     filesystem.add_option('--write-thumbnail',
 513             action='store_true', dest='writethumbnail',
 514             help='write thumbnail image to disk', default=False)
 515     filesystem.add_option('--load-info',
 516             dest='load_info_filename', metavar='FILE',
 517             help='json file containing the video information (created with the "--write-json" option)')
 518     filesystem.add_option('--cookies',
 519             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 520     filesystem.add_option(
 521         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 522         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 523     filesystem.add_option(
 524         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 525         help='Disable filesystem caching')
 526     filesystem.add_option(
 527         '--rm-cache-dir', action='store_true', dest='rm_cachedir',
 528         help='Delete all filesystem cache files')
 529
 530
 531     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 532             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 533     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 534             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 535     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 536             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 537     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 538             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
 539     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 540             help='keeps the video file on disk after the post-processing; the video is erased by default')
 541     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 542             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 543     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 544             help='embed subtitles in the video (only for mp4 videos)')
 545     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 546             help='embed thumbnail in the audio as cover art')
 547     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 548             help='write metadata to the video file')
 549     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 550             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 551     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 552         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 553     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 554         help='Prefer ffmpeg over avconv for running the postprocessors')
 555     postproc.add_option('--exec', metavar='', action='store', dest='execstring',
 556         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Must be enclosed in quotes. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'' )
 557
 558     parser.add_option_group(general)
 559     parser.add_option_group(selection)
 560     parser.add_option_group(downloader)
 561     parser.add_option_group(filesystem)
 562     parser.add_option_group(verbosity)
 563     parser.add_option_group(workarounds)
 564     parser.add_option_group(video_format)
 565     parser.add_option_group(subtitles)
 566     parser.add_option_group(authentication)
 567     parser.add_option_group(postproc)
 568
 569     if overrideArguments is not None:
 570         opts, args = parser.parse_args(overrideArguments)
 571         if opts.verbose:
 572             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 573     else:
 574         commandLineConf = sys.argv[1:]
 575         if '--ignore-config' in commandLineConf:
 576             systemConf = []
 577             userConf = []
 578         else:
 579             systemConf = _readOptions('/etc/youtube-dl.conf')
 580             if '--ignore-config' in systemConf:
 581                 userConf = []
 582             else:
 583                 userConf = _readUserConf()
 584         argv = systemConf + userConf + commandLineConf
 585
 586         opts, args = parser.parse_args(argv)
 587         if opts.verbose:
 588             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 589             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 590             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 591
 592     return parser, opts, args
 593
 594
 595 def _real_main(argv=None):
 596     # Compatibility fixes for Windows
 597     if sys.platform == 'win32':
 598         # https://github.com/rg3/youtube-dl/issues/820
 599         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 600
 601     setproctitle(u'youtube-dl')
 602
 603     parser, opts, args = parseOpts(argv)
 604
 605     # Set user agent
 606     if opts.user_agent is not None:
 607         std_headers['User-Agent'] = opts.user_agent
 608
 609     # Set referer
 610     if opts.referer is not None:
 611         std_headers['Referer'] = opts.referer
 612
 613     # Custom HTTP headers
 614     if opts.headers is not None:
 615         for h in opts.headers:
 616             if h.find(':', 1) < 0:
 617                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 618             key, value = h.split(':', 2)
 619             if opts.verbose:
 620                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 621             std_headers[key] = value
 622
 623     # Dump user agent
 624     if opts.dump_user_agent:
 625         compat_print(std_headers['User-Agent'])
 626         sys.exit(0)
 627
 628     # Batch file verification
 629     batch_urls = []
 630     if opts.batchfile is not None:
 631         try:
 632             if opts.batchfile == '-':
 633                 batchfd = sys.stdin
 634             else:
 635                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 636             batch_urls = read_batch_urls(batchfd)
 637             if opts.verbose:
 638                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 639         except IOError:
 640             sys.exit(u'ERROR: batch file could not be read')
 641     all_urls = batch_urls + args
 642     all_urls = [url.strip() for url in all_urls]
 643     _enc = preferredencoding()
 644     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 645
 646     extractors = gen_extractors()
 647
 648     if opts.list_extractors:
 649         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 650             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 651             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 652             for mu in matchedUrls:
 653                 compat_print(u'  ' + mu)
 654         sys.exit(0)
 655     if opts.list_extractor_descriptions:
 656         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 657             if not ie._WORKING:
 658                 continue
 659             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 660             if desc is False:
 661                 continue
 662             if hasattr(ie, 'SEARCH_KEY'):
 663                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 664                 _COUNTS = (u'', u'5', u'10', u'all')
 665                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 666             compat_print(desc)
 667         sys.exit(0)
 668
 669
 670     # Conflicting, missing and erroneous options
 671     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 672         parser.error(u'using .netrc conflicts with giving username/password')
 673     if opts.password is not None and opts.username is None:
 674         parser.error(u'account username missing\n')
 675     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 676         parser.error(u'using output template conflicts with using title, video ID or auto number')
 677     if opts.usetitle and opts.useid:
 678         parser.error(u'using title conflicts with using video ID')
 679     if opts.username is not None and opts.password is None:
 680         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 681     if opts.ratelimit is not None:
 682         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 683         if numeric_limit is None:
 684             parser.error(u'invalid rate limit specified')
 685         opts.ratelimit = numeric_limit
 686     if opts.min_filesize is not None:
 687         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 688         if numeric_limit is None:
 689             parser.error(u'invalid min_filesize specified')
 690         opts.min_filesize = numeric_limit
 691     if opts.max_filesize is not None:
 692         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 693         if numeric_limit is None:
 694             parser.error(u'invalid max_filesize specified')
 695         opts.max_filesize = numeric_limit
 696     if opts.retries is not None:
 697         try:
 698             opts.retries = int(opts.retries)
 699         except (TypeError, ValueError):
 700             parser.error(u'invalid retry count specified')
 701     if opts.buffersize is not None:
 702         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 703         if numeric_buffersize is None:
 704             parser.error(u'invalid buffer size specified')
 705         opts.buffersize = numeric_buffersize
 706     if opts.playliststart <= 0:
 707         raise ValueError(u'Playlist start must be positive')
 708     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 709         raise ValueError(u'Playlist end must be greater than playlist start')
 710     if opts.extractaudio:
 711         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 712             parser.error(u'invalid audio format specified')
 713     if opts.audioquality:
 714         opts.audioquality = opts.audioquality.strip('k').strip('K')
 715         if not opts.audioquality.isdigit():
 716             parser.error(u'invalid audio quality specified')
 717     if opts.recodevideo is not None:
 718         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 719             parser.error(u'invalid video recode format specified')
 720     if opts.date is not None:
 721         date = DateRange.day(opts.date)
 722     else:
 723         date = DateRange(opts.dateafter, opts.datebefore)
 724     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 725         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 726
 727     # Do not download videos when there are audio-only formats
 728     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 729         opts.format = 'bestaudio/best'
 730
 731     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 732     # this was the old behaviour if only --all-sub was given.
 733     if opts.allsubtitles and (opts.writeautomaticsub == False):
 734         opts.writesubtitles = True
 735
 736     if sys.version_info < (3,):
 737         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 738         if opts.outtmpl is not None:
 739             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 740     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 741             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 742             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 743             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 744             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 745             or (opts.useid and u'%(id)s.%(ext)s')
 746             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 747             or DEFAULT_OUTTMPL)
 748     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 749         parser.error(u'Cannot download a video and extract audio into the same'
 750                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 751                      u' template'.format(outtmpl))
 752
 753     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 754     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 755
 756     ydl_opts = {
 757         'usenetrc': opts.usenetrc,
 758         'username': opts.username,
 759         'password': opts.password,
 760         'twofactor': opts.twofactor,
 761         'videopassword': opts.videopassword,
 762         'quiet': (opts.quiet or any_printing),
 763         'no_warnings': opts.no_warnings,
 764         'forceurl': opts.geturl,
 765         'forcetitle': opts.gettitle,
 766         'forceid': opts.getid,
 767         'forcethumbnail': opts.getthumbnail,
 768         'forcedescription': opts.getdescription,
 769         'forceduration': opts.getduration,
 770         'forcefilename': opts.getfilename,
 771         'forceformat': opts.getformat,
 772         'forcejson': opts.dumpjson,
 773         'simulate': opts.simulate,
 774         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 775         'format': opts.format,
 776         'format_limit': opts.format_limit,
 777         'listformats': opts.listformats,
 778         'outtmpl': outtmpl,
 779         'autonumber_size': opts.autonumber_size,
 780         'restrictfilenames': opts.restrictfilenames,
 781         'ignoreerrors': opts.ignoreerrors,
 782         'ratelimit': opts.ratelimit,
 783         'nooverwrites': opts.nooverwrites,
 784         'retries': opts.retries,
 785         'buffersize': opts.buffersize,
 786         'noresizebuffer': opts.noresizebuffer,
 787         'continuedl': opts.continue_dl,
 788         'noprogress': opts.noprogress,
 789         'progress_with_newline': opts.progress_with_newline,
 790         'playliststart': opts.playliststart,
 791         'playlistend': opts.playlistend,
 792         'noplaylist': opts.noplaylist,
 793         'logtostderr': opts.outtmpl == '-',
 794         'consoletitle': opts.consoletitle,
 795         'nopart': opts.nopart,
 796         'updatetime': opts.updatetime,
 797         'writedescription': opts.writedescription,
 798         'writeannotations': opts.writeannotations,
 799         'writeinfojson': opts.writeinfojson,
 800         'writethumbnail': opts.writethumbnail,
 801         'writesubtitles': opts.writesubtitles,
 802         'writeautomaticsub': opts.writeautomaticsub,
 803         'allsubtitles': opts.allsubtitles,
 804         'listsubtitles': opts.listsubtitles,
 805         'subtitlesformat': opts.subtitlesformat,
 806         'subtitleslangs': opts.subtitleslangs,
 807         'matchtitle': decodeOption(opts.matchtitle),
 808         'rejecttitle': decodeOption(opts.rejecttitle),
 809         'max_downloads': opts.max_downloads,
 810         'prefer_free_formats': opts.prefer_free_formats,
 811         'verbose': opts.verbose,
 812         'dump_intermediate_pages': opts.dump_intermediate_pages,
 813         'write_pages': opts.write_pages,
 814         'test': opts.test,
 815         'keepvideo': opts.keepvideo,
 816         'min_filesize': opts.min_filesize,
 817         'max_filesize': opts.max_filesize,
 818         'min_views': opts.min_views,
 819         'max_views': opts.max_views,
 820         'daterange': date,
 821         'cachedir': opts.cachedir,
 822         'youtube_print_sig_code': opts.youtube_print_sig_code,
 823         'age_limit': opts.age_limit,
 824         'download_archive': download_archive_fn,
 825         'cookiefile': opts.cookiefile,
 826         'nocheckcertificate': opts.no_check_certificate,
 827         'prefer_insecure': opts.prefer_insecure,
 828         'proxy': opts.proxy,
 829         'socket_timeout': opts.socket_timeout,
 830         'bidi_workaround': opts.bidi_workaround,
 831         'debug_printtraffic': opts.debug_printtraffic,
 832         'prefer_ffmpeg': opts.prefer_ffmpeg,
 833         'include_ads': opts.include_ads,
 834         'default_search': opts.default_search,
 835         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 836         'encoding': opts.encoding,
 837         'execstring': opts.execstring,
 838     }
 839
 840     with YoutubeDL(ydl_opts) as ydl:
 841         ydl.print_debug_header()
 842         ydl.add_default_info_extractors()
 843
 844         # PostProcessors
 845         # Add the metadata pp first, the other pps will copy it
 846         if opts.addmetadata:
 847             ydl.add_post_processor(FFmpegMetadataPP())
 848         if opts.extractaudio:
 849             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 850         if opts.recodevideo:
 851             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 852         if opts.embedsubtitles:
 853             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 854         if opts.xattrs:
 855             ydl.add_post_processor(XAttrMetadataPP())
 856         if opts.embedthumbnail:
 857             if not opts.addmetadata:
 858                 ydl.add_post_processor(FFmpegAudioFixPP())
 859             ydl.add_post_processor(AtomicParsleyPP())
 860
 861
 862         # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
 863         # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
 864         if opts.execstring:
 865             ydl.add_post_processor(ExecAfterDownloadPP(verboseOutput=opts.verbose,commandString=opts.execstring))
 866
 867         # Update version
 868         if opts.update_self:
 869             update_self(ydl.to_screen, opts.verbose)
 870
 871         # Remove cache dir
 872         if opts.rm_cachedir:
 873             if opts.cachedir is None:
 874                 ydl.to_screen(u'No cache dir specified (Did you combine --no-cache-dir and --rm-cache-dir?)')
 875             else:
 876                 if ('.cache' not in opts.cachedir) or ('youtube-dl' not in opts.cachedir):
 877                     ydl.to_screen(u'Not removing directory %s - this does not look like a cache dir')
 878                     retcode = 141
 879                 else:
 880                     ydl.to_screen(
 881                         u'Removing cache dir %s .' % opts.cachedir,
 882                         skip_eol=True)
 883                     if os.path.exists(opts.cachedir):
 884                         ydl.to_screen(u'.', skip_eol=True)
 885                         shutil.rmtree(opts.cachedir)
 886                     ydl.to_screen(u'.')
 887
 888         # Maybe do nothing
 889         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 890             if not (opts.update_self or opts.rm_cachedir):
 891                 parser.error(u'you must provide at least one URL')
 892             else:
 893                 sys.exit()
 894
 895         try:
 896             if opts.load_info_filename is not None:
 897                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 898             else:
 899                 retcode = ydl.download(all_urls)
 900         except MaxDownloadsReached:
 901             ydl.to_screen(u'--max-download limit reached, aborting.')
 902             retcode = 101
 903
 904     sys.exit(retcode)
 905
 906
 907 def main(argv=None):
 908     try:
 909         _real_main(argv)
 910     except DownloadError:
 911         sys.exit(1)
 912     except SameFileError:
 913         sys.exit(u'ERROR: fixed output name but more than one file to download')
 914     except KeyboardInterrupt:
 915         sys.exit(u'\nERROR: Interrupted by user')