youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70     'David Fabijan',
  71     'Sebastian Haas',
  72     'Alexander Kirk',
  73     'Erik Johnson',
  74     'Keith Beckman',
  75     'Ole Ernst',
  76     'Aaron McDaniel (mcd1992)',
  77 )
  78
  79 __license__ = 'Public Domain'
  80
  81 import codecs
  82 import io
  83 import optparse
  84 import os
  85 import random
  86 import shlex
  87 import sys
  88
  89
  90 from .utils import (
  91     compat_getpass,
  92     compat_print,
  93     DateRange,
  94     DEFAULT_OUTTMPL,
  95     decodeOption,
  96     get_term_width,
  97     DownloadError,
  98     MaxDownloadsReached,
  99     preferredencoding,
 100     read_batch_urls,
 101     SameFileError,
 102     setproctitle,
 103     std_headers,
 104     write_string,
 105 )
 106 from .update import update_self
 107 from .downloader import (
 108     FileDownloader,
 109 )
 110 from .extractor import gen_extractors
 111 from .version import __version__
 112 from .YoutubeDL import YoutubeDL
 113 from .postprocessor import (
 114     AtomicParsleyPP,
 115     FFmpegAudioFixPP,
 116     FFmpegMetadataPP,
 117     FFmpegVideoConvertor,
 118     FFmpegExtractAudioPP,
 119     FFmpegEmbedSubtitlePP,
 120     XAttrMetadataPP,
 121     ExecAfterDownloadPP,
 122 )
 123
 124
 125 def parseOpts(overrideArguments=None):
 126     def _readOptions(filename_bytes, default=[]):
 127         try:
 128             optionf = open(filename_bytes)
 129         except IOError:
 130             return default  # silently skip if file is not present
 131         try:
 132             res = []
 133             for l in optionf:
 134                 res += shlex.split(l, comments=True)
 135         finally:
 136             optionf.close()
 137         return res
 138
 139     def _readUserConf():
 140         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 141         if xdg_config_home:
 142             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 143             if not os.path.isfile(userConfFile):
 144                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 145         else:
 146             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 147             if not os.path.isfile(userConfFile):
 148                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 149         userConf = _readOptions(userConfFile, None)
 150
 151         if userConf is None:
 152             appdata_dir = os.environ.get('appdata')
 153             if appdata_dir:
 154                 userConf = _readOptions(
 155                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 156                     default=None)
 157                 if userConf is None:
 158                     userConf = _readOptions(
 159                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 160                         default=None)
 161
 162         if userConf is None:
 163             userConf = _readOptions(
 164                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 165                 default=None)
 166         if userConf is None:
 167             userConf = _readOptions(
 168                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 169                 default=None)
 170
 171         if userConf is None:
 172             userConf = []
 173
 174         return userConf
 175
 176     def _format_option_string(option):
 177         ''' ('-o', '--option') -> -o, --format METAVAR'''
 178
 179         opts = []
 180
 181         if option._short_opts:
 182             opts.append(option._short_opts[0])
 183         if option._long_opts:
 184             opts.append(option._long_opts[0])
 185         if len(opts) > 1:
 186             opts.insert(1, ', ')
 187
 188         if option.takes_value(): opts.append(' %s' % option.metavar)
 189
 190         return "".join(opts)
 191
 192     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 193         setattr(parser.values, option.dest, value.split(','))
 194
 195     def _hide_login_info(opts):
 196         opts = list(opts)
 197         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 198             try:
 199                 i = opts.index(private_opt)
 200                 opts[i+1] = '<PRIVATE>'
 201             except ValueError:
 202                 pass
 203         return opts
 204
 205     max_width = 80
 206     max_help_position = 80
 207
 208     # No need to wrap help messages if we're on a wide console
 209     columns = get_term_width()
 210     if columns: max_width = columns
 211
 212     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 213     fmt.format_option_strings = _format_option_string
 214
 215     kw = {
 216         'version'   : __version__,
 217         'formatter' : fmt,
 218         'usage' : '%prog [options] url [url...]',
 219         'conflict_handler' : 'resolve',
 220     }
 221
 222     parser = optparse.OptionParser(**kw)
 223
 224     # option groups
 225     general        = optparse.OptionGroup(parser, 'General Options')
 226     selection      = optparse.OptionGroup(parser, 'Video Selection')
 227     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 228     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 229     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 230     downloader     = optparse.OptionGroup(parser, 'Download Options')
 231     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 232     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 233     workarounds    = optparse.OptionGroup(parser, 'Workarounds')
 234     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 235
 236     general.add_option('-h', '--help',
 237             action='help', help='print this help text and exit')
 238     general.add_option('-v', '--version',
 239             action='version', help='print program version and exit')
 240     general.add_option('-U', '--update',
 241             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 242     general.add_option('-i', '--ignore-errors',
 243             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 244     general.add_option('--abort-on-error',
 245             action='store_false', dest='ignoreerrors',
 246             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 247     general.add_option('--dump-user-agent',
 248             action='store_true', dest='dump_user_agent',
 249             help='display the current browser identification', default=False)
 250     general.add_option('--list-extractors',
 251             action='store_true', dest='list_extractors',
 252             help='List all supported extractors and the URLs they would handle', default=False)
 253     general.add_option('--extractor-descriptions',
 254             action='store_true', dest='list_extractor_descriptions',
 255             help='Output descriptions of all supported extractors', default=False)
 256     general.add_option(
 257         '--proxy', dest='proxy', default=None, metavar='URL',
 258         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 259     general.add_option(
 260         '--socket-timeout', dest='socket_timeout',
 261         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 262     general.add_option(
 263         '--default-search',
 264         dest='default_search', metavar='PREFIX',
 265         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
 266     general.add_option(
 267         '--ignore-config',
 268         action='store_true',
 269         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 270
 271     selection.add_option(
 272         '--playlist-start',
 273         dest='playliststart', metavar='NUMBER', default=1, type=int,
 274         help='playlist video to start at (default is %default)')
 275     selection.add_option(
 276         '--playlist-end',
 277         dest='playlistend', metavar='NUMBER', default=None, type=int,
 278         help='playlist video to end at (default is last)')
 279     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 280     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 281     selection.add_option('--max-downloads', metavar='NUMBER',
 282                          dest='max_downloads', type=int, default=None,
 283                          help='Abort after downloading NUMBER files')
 284     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 285     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 286     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 287     selection.add_option(
 288         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 289         help='download only videos uploaded on or before this date (i.e. inclusive)')
 290     selection.add_option(
 291         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 292         help='download only videos uploaded on or after this date (i.e. inclusive)')
 293     selection.add_option(
 294         '--min-views', metavar='COUNT', dest='min_views',
 295         default=None, type=int,
 296         help="Do not download any videos with less than COUNT views",)
 297     selection.add_option(
 298         '--max-views', metavar='COUNT', dest='max_views',
 299         default=None, type=int,
 300         help="Do not download any videos with more than COUNT views",)
 301     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 302     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 303                          help='download only videos suitable for the given age',
 304                          default=None, type=int)
 305     selection.add_option('--download-archive', metavar='FILE',
 306                          dest='download_archive',
 307                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 308     selection.add_option(
 309         '--include-ads', dest='include_ads',
 310         action='store_true',
 311         help='Download advertisements as well (experimental)')
 312     selection.add_option(
 313         '--youtube-include-dash-manifest', action='store_true',
 314         dest='youtube_include_dash_manifest', default=False,
 315         help='Try to download the DASH manifest on YouTube videos (experimental)')
 316
 317     authentication.add_option('-u', '--username',
 318             dest='username', metavar='USERNAME', help='account username')
 319     authentication.add_option('-p', '--password',
 320             dest='password', metavar='PASSWORD', help='account password')
 321     authentication.add_option('-2', '--twofactor',
 322             dest='twofactor', metavar='TWOFACTOR', help='two-factor auth code')
 323     authentication.add_option('-n', '--netrc',
 324             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 325     authentication.add_option('--video-password',
 326             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 327
 328
 329     video_format.add_option('-f', '--format',
 330             action='store', dest='format', metavar='FORMAT', default=None,
 331             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 332     video_format.add_option('--all-formats',
 333             action='store_const', dest='format', help='download all available video formats', const='all')
 334     video_format.add_option('--prefer-free-formats',
 335             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 336     video_format.add_option('--max-quality',
 337             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 338     video_format.add_option('-F', '--list-formats',
 339             action='store_true', dest='listformats', help='list all available formats')
 340
 341     subtitles.add_option('--write-sub', '--write-srt',
 342             action='store_true', dest='writesubtitles',
 343             help='write subtitle file', default=False)
 344     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 345             action='store_true', dest='writeautomaticsub',
 346             help='write automatic subtitle file (youtube only)', default=False)
 347     subtitles.add_option('--all-subs',
 348             action='store_true', dest='allsubtitles',
 349             help='downloads all the available subtitles of the video', default=False)
 350     subtitles.add_option('--list-subs',
 351             action='store_true', dest='listsubtitles',
 352             help='lists all available subtitles for the video', default=False)
 353     subtitles.add_option('--sub-format',
 354             action='store', dest='subtitlesformat', metavar='FORMAT',
 355             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 356     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 357             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 358             default=[], callback=_comma_separated_values_options_callback,
 359             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 360
 361     downloader.add_option('-r', '--rate-limit',
 362             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 363     downloader.add_option('-R', '--retries',
 364             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 365     downloader.add_option('--buffer-size',
 366             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 367     downloader.add_option('--no-resize-buffer',
 368             action='store_true', dest='noresizebuffer',
 369             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 370     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 371
 372     workarounds.add_option(
 373         '--encoding', dest='encoding', metavar='ENCODING',
 374         help='Force the specified encoding (experimental)')
 375     workarounds.add_option(
 376         '--no-check-certificate', action='store_true',
 377         dest='no_check_certificate', default=False,
 378         help='Suppress HTTPS certificate validation.')
 379     workarounds.add_option(
 380         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 381         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 382     workarounds.add_option(
 383         '--user-agent', metavar='UA',
 384         dest='user_agent', help='specify a custom user agent')
 385     workarounds.add_option(
 386         '--referer', metavar='REF',
 387         dest='referer', default=None,
 388         help='specify a custom referer, use if the video access is restricted to one domain',
 389     )
 390     workarounds.add_option(
 391         '--add-header', metavar='FIELD:VALUE',
 392         dest='headers', action='append',
 393         help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
 394     )
 395     workarounds.add_option(
 396         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 397         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 398
 399     verbosity.add_option('-q', '--quiet',
 400             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 401     verbosity.add_option(
 402         '--no-warnings',
 403         dest='no_warnings', action='store_true', default=False,
 404         help='Ignore warnings')
 405     verbosity.add_option('-s', '--simulate',
 406             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 407     verbosity.add_option('--skip-download',
 408             action='store_true', dest='skip_download', help='do not download the video', default=False)
 409     verbosity.add_option('-g', '--get-url',
 410             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 411     verbosity.add_option('-e', '--get-title',
 412             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 413     verbosity.add_option('--get-id',
 414             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 415     verbosity.add_option('--get-thumbnail',
 416             action='store_true', dest='getthumbnail',
 417             help='simulate, quiet but print thumbnail URL', default=False)
 418     verbosity.add_option('--get-description',
 419             action='store_true', dest='getdescription',
 420             help='simulate, quiet but print video description', default=False)
 421     verbosity.add_option('--get-duration',
 422             action='store_true', dest='getduration',
 423             help='simulate, quiet but print video length', default=False)
 424     verbosity.add_option('--get-filename',
 425             action='store_true', dest='getfilename',
 426             help='simulate, quiet but print output filename', default=False)
 427     verbosity.add_option('--get-format',
 428             action='store_true', dest='getformat',
 429             help='simulate, quiet but print output format', default=False)
 430     verbosity.add_option('-j', '--dump-json',
 431             action='store_true', dest='dumpjson',
 432             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 433     verbosity.add_option('--newline',
 434             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 435     verbosity.add_option('--no-progress',
 436             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 437     verbosity.add_option('--console-title',
 438             action='store_true', dest='consoletitle',
 439             help='display progress in console titlebar', default=False)
 440     verbosity.add_option('-v', '--verbose',
 441             action='store_true', dest='verbose', help='print various debugging information', default=False)
 442     verbosity.add_option('--dump-intermediate-pages',
 443             action='store_true', dest='dump_intermediate_pages', default=False,
 444             help='print downloaded pages to debug problems (very verbose)')
 445     verbosity.add_option('--write-pages',
 446             action='store_true', dest='write_pages', default=False,
 447             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 448     verbosity.add_option('--youtube-print-sig-code',
 449             action='store_true', dest='youtube_print_sig_code', default=False,
 450             help=optparse.SUPPRESS_HELP)
 451     verbosity.add_option('--print-traffic',
 452             dest='debug_printtraffic', action='store_true', default=False,
 453             help='Display sent and read HTTP traffic')
 454
 455
 456     filesystem.add_option('-a', '--batch-file',
 457             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 458     filesystem.add_option('--id',
 459             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 460     filesystem.add_option('-A', '--auto-number',
 461             action='store_true', dest='autonumber',
 462             help='number downloaded files starting from 00000', default=False)
 463     filesystem.add_option('-o', '--output',
 464             dest='outtmpl', metavar='TEMPLATE',
 465             help=('output filename template. Use %(title)s to get the title, '
 466                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 467                   '%(autonumber)s to get an automatically incremented number, '
 468                   '%(ext)s for the filename extension, '
 469                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 470                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 471                   '%(upload_date)s for the upload date (YYYYMMDD), '
 472                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 473                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 474                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 475                   '%(height)s and %(width)s for the width and height of the video format. '
 476                   '%(resolution)s for a textual description of the resolution of the video format. '
 477                   'Use - to output to stdout. Can also be used to download to a different directory, '
 478                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 479     filesystem.add_option('--autonumber-size',
 480             dest='autonumber_size', metavar='NUMBER',
 481             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 482     filesystem.add_option('--restrict-filenames',
 483             action='store_true', dest='restrictfilenames',
 484             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 485     filesystem.add_option('-t', '--title',
 486             action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
 487     filesystem.add_option('-l', '--literal',
 488             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 489     filesystem.add_option('-w', '--no-overwrites',
 490             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 491     filesystem.add_option('-c', '--continue',
 492             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 493     filesystem.add_option('--no-continue',
 494             action='store_false', dest='continue_dl',
 495             help='do not resume partially downloaded files (restart from beginning)')
 496     filesystem.add_option('--no-part',
 497             action='store_true', dest='nopart', help='do not use .part files', default=False)
 498     filesystem.add_option('--no-mtime',
 499             action='store_false', dest='updatetime',
 500             help='do not use the Last-modified header to set the file modification time', default=True)
 501     filesystem.add_option('--write-description',
 502             action='store_true', dest='writedescription',
 503             help='write video description to a .description file', default=False)
 504     filesystem.add_option('--write-info-json',
 505             action='store_true', dest='writeinfojson',
 506             help='write video metadata to a .info.json file', default=False)
 507     filesystem.add_option('--write-annotations',
 508             action='store_true', dest='writeannotations',
 509             help='write video annotations to a .annotation file', default=False)
 510     filesystem.add_option('--write-thumbnail',
 511             action='store_true', dest='writethumbnail',
 512             help='write thumbnail image to disk', default=False)
 513     filesystem.add_option('--load-info',
 514             dest='load_info_filename', metavar='FILE',
 515             help='json file containing the video information (created with the "--write-json" option)')
 516     filesystem.add_option('--cookies',
 517             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 518     filesystem.add_option(
 519         '--cache-dir', dest='cachedir', default=None, metavar='DIR',
 520         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 521     filesystem.add_option(
 522         '--no-cache-dir', action='store_const', const=False, dest='cachedir',
 523         help='Disable filesystem caching')
 524     filesystem.add_option(
 525         '--rm-cache-dir', action='store_true', dest='rm_cachedir',
 526         help='Delete all filesystem cache files')
 527
 528
 529     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 530             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 531     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 532             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 533     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 534             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 535     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 536             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
 537     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 538             help='keeps the video file on disk after the post-processing; the video is erased by default')
 539     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 540             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 541     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 542             help='embed subtitles in the video (only for mp4 videos)')
 543     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 544             help='embed thumbnail in the audio as cover art')
 545     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 546             help='write metadata to the video file')
 547     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 548             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 549     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 550         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 551     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 552         help='Prefer ffmpeg over avconv for running the postprocessors')
 553     postproc.add_option(
 554         '--exec', metavar='CMD', dest='exec_cmd',
 555         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'' )
 556
 557     parser.add_option_group(general)
 558     parser.add_option_group(selection)
 559     parser.add_option_group(downloader)
 560     parser.add_option_group(filesystem)
 561     parser.add_option_group(verbosity)
 562     parser.add_option_group(workarounds)
 563     parser.add_option_group(video_format)
 564     parser.add_option_group(subtitles)
 565     parser.add_option_group(authentication)
 566     parser.add_option_group(postproc)
 567
 568     if overrideArguments is not None:
 569         opts, args = parser.parse_args(overrideArguments)
 570         if opts.verbose:
 571             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 572     else:
 573         commandLineConf = sys.argv[1:]
 574         if '--ignore-config' in commandLineConf:
 575             systemConf = []
 576             userConf = []
 577         else:
 578             systemConf = _readOptions('/etc/youtube-dl.conf')
 579             if '--ignore-config' in systemConf:
 580                 userConf = []
 581             else:
 582                 userConf = _readUserConf()
 583         argv = systemConf + userConf + commandLineConf
 584
 585         opts, args = parser.parse_args(argv)
 586         if opts.verbose:
 587             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 588             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 589             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 590
 591     return parser, opts, args
 592
 593
 594 def _real_main(argv=None):
 595     # Compatibility fixes for Windows
 596     if sys.platform == 'win32':
 597         # https://github.com/rg3/youtube-dl/issues/820
 598         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 599
 600     setproctitle(u'youtube-dl')
 601
 602     parser, opts, args = parseOpts(argv)
 603
 604     # Set user agent
 605     if opts.user_agent is not None:
 606         std_headers['User-Agent'] = opts.user_agent
 607
 608     # Set referer
 609     if opts.referer is not None:
 610         std_headers['Referer'] = opts.referer
 611
 612     # Custom HTTP headers
 613     if opts.headers is not None:
 614         for h in opts.headers:
 615             if h.find(':', 1) < 0:
 616                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 617             key, value = h.split(':', 2)
 618             if opts.verbose:
 619                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 620             std_headers[key] = value
 621
 622     # Dump user agent
 623     if opts.dump_user_agent:
 624         compat_print(std_headers['User-Agent'])
 625         sys.exit(0)
 626
 627     # Batch file verification
 628     batch_urls = []
 629     if opts.batchfile is not None:
 630         try:
 631             if opts.batchfile == '-':
 632                 batchfd = sys.stdin
 633             else:
 634                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 635             batch_urls = read_batch_urls(batchfd)
 636             if opts.verbose:
 637                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 638         except IOError:
 639             sys.exit(u'ERROR: batch file could not be read')
 640     all_urls = batch_urls + args
 641     all_urls = [url.strip() for url in all_urls]
 642     _enc = preferredencoding()
 643     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 644
 645     extractors = gen_extractors()
 646
 647     if opts.list_extractors:
 648         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 649             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 650             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 651             for mu in matchedUrls:
 652                 compat_print(u'  ' + mu)
 653         sys.exit(0)
 654     if opts.list_extractor_descriptions:
 655         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 656             if not ie._WORKING:
 657                 continue
 658             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 659             if desc is False:
 660                 continue
 661             if hasattr(ie, 'SEARCH_KEY'):
 662                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 663                 _COUNTS = (u'', u'5', u'10', u'all')
 664                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 665             compat_print(desc)
 666         sys.exit(0)
 667
 668
 669     # Conflicting, missing and erroneous options
 670     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 671         parser.error(u'using .netrc conflicts with giving username/password')
 672     if opts.password is not None and opts.username is None:
 673         parser.error(u'account username missing\n')
 674     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 675         parser.error(u'using output template conflicts with using title, video ID or auto number')
 676     if opts.usetitle and opts.useid:
 677         parser.error(u'using title conflicts with using video ID')
 678     if opts.username is not None and opts.password is None:
 679         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 680     if opts.ratelimit is not None:
 681         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 682         if numeric_limit is None:
 683             parser.error(u'invalid rate limit specified')
 684         opts.ratelimit = numeric_limit
 685     if opts.min_filesize is not None:
 686         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 687         if numeric_limit is None:
 688             parser.error(u'invalid min_filesize specified')
 689         opts.min_filesize = numeric_limit
 690     if opts.max_filesize is not None:
 691         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 692         if numeric_limit is None:
 693             parser.error(u'invalid max_filesize specified')
 694         opts.max_filesize = numeric_limit
 695     if opts.retries is not None:
 696         try:
 697             opts.retries = int(opts.retries)
 698         except (TypeError, ValueError):
 699             parser.error(u'invalid retry count specified')
 700     if opts.buffersize is not None:
 701         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 702         if numeric_buffersize is None:
 703             parser.error(u'invalid buffer size specified')
 704         opts.buffersize = numeric_buffersize
 705     if opts.playliststart <= 0:
 706         raise ValueError(u'Playlist start must be positive')
 707     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 708         raise ValueError(u'Playlist end must be greater than playlist start')
 709     if opts.extractaudio:
 710         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 711             parser.error(u'invalid audio format specified')
 712     if opts.audioquality:
 713         opts.audioquality = opts.audioquality.strip('k').strip('K')
 714         if not opts.audioquality.isdigit():
 715             parser.error(u'invalid audio quality specified')
 716     if opts.recodevideo is not None:
 717         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 718             parser.error(u'invalid video recode format specified')
 719     if opts.date is not None:
 720         date = DateRange.day(opts.date)
 721     else:
 722         date = DateRange(opts.dateafter, opts.datebefore)
 723     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 724         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 725
 726     # Do not download videos when there are audio-only formats
 727     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 728         opts.format = 'bestaudio/best'
 729
 730     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 731     # this was the old behaviour if only --all-sub was given.
 732     if opts.allsubtitles and (opts.writeautomaticsub == False):
 733         opts.writesubtitles = True
 734
 735     if sys.version_info < (3,):
 736         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 737         if opts.outtmpl is not None:
 738             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 739     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 740             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 741             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 742             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 743             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 744             or (opts.useid and u'%(id)s.%(ext)s')
 745             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 746             or DEFAULT_OUTTMPL)
 747     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 748         parser.error(u'Cannot download a video and extract audio into the same'
 749                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 750                      u' template'.format(outtmpl))
 751
 752     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 753     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 754
 755     ydl_opts = {
 756         'usenetrc': opts.usenetrc,
 757         'username': opts.username,
 758         'password': opts.password,
 759         'twofactor': opts.twofactor,
 760         'videopassword': opts.videopassword,
 761         'quiet': (opts.quiet or any_printing),
 762         'no_warnings': opts.no_warnings,
 763         'forceurl': opts.geturl,
 764         'forcetitle': opts.gettitle,
 765         'forceid': opts.getid,
 766         'forcethumbnail': opts.getthumbnail,
 767         'forcedescription': opts.getdescription,
 768         'forceduration': opts.getduration,
 769         'forcefilename': opts.getfilename,
 770         'forceformat': opts.getformat,
 771         'forcejson': opts.dumpjson,
 772         'simulate': opts.simulate,
 773         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 774         'format': opts.format,
 775         'format_limit': opts.format_limit,
 776         'listformats': opts.listformats,
 777         'outtmpl': outtmpl,
 778         'autonumber_size': opts.autonumber_size,
 779         'restrictfilenames': opts.restrictfilenames,
 780         'ignoreerrors': opts.ignoreerrors,
 781         'ratelimit': opts.ratelimit,
 782         'nooverwrites': opts.nooverwrites,
 783         'retries': opts.retries,
 784         'buffersize': opts.buffersize,
 785         'noresizebuffer': opts.noresizebuffer,
 786         'continuedl': opts.continue_dl,
 787         'noprogress': opts.noprogress,
 788         'progress_with_newline': opts.progress_with_newline,
 789         'playliststart': opts.playliststart,
 790         'playlistend': opts.playlistend,
 791         'noplaylist': opts.noplaylist,
 792         'logtostderr': opts.outtmpl == '-',
 793         'consoletitle': opts.consoletitle,
 794         'nopart': opts.nopart,
 795         'updatetime': opts.updatetime,
 796         'writedescription': opts.writedescription,
 797         'writeannotations': opts.writeannotations,
 798         'writeinfojson': opts.writeinfojson,
 799         'writethumbnail': opts.writethumbnail,
 800         'writesubtitles': opts.writesubtitles,
 801         'writeautomaticsub': opts.writeautomaticsub,
 802         'allsubtitles': opts.allsubtitles,
 803         'listsubtitles': opts.listsubtitles,
 804         'subtitlesformat': opts.subtitlesformat,
 805         'subtitleslangs': opts.subtitleslangs,
 806         'matchtitle': decodeOption(opts.matchtitle),
 807         'rejecttitle': decodeOption(opts.rejecttitle),
 808         'max_downloads': opts.max_downloads,
 809         'prefer_free_formats': opts.prefer_free_formats,
 810         'verbose': opts.verbose,
 811         'dump_intermediate_pages': opts.dump_intermediate_pages,
 812         'write_pages': opts.write_pages,
 813         'test': opts.test,
 814         'keepvideo': opts.keepvideo,
 815         'min_filesize': opts.min_filesize,
 816         'max_filesize': opts.max_filesize,
 817         'min_views': opts.min_views,
 818         'max_views': opts.max_views,
 819         'daterange': date,
 820         'cachedir': opts.cachedir,
 821         'youtube_print_sig_code': opts.youtube_print_sig_code,
 822         'age_limit': opts.age_limit,
 823         'download_archive': download_archive_fn,
 824         'cookiefile': opts.cookiefile,
 825         'nocheckcertificate': opts.no_check_certificate,
 826         'prefer_insecure': opts.prefer_insecure,
 827         'proxy': opts.proxy,
 828         'socket_timeout': opts.socket_timeout,
 829         'bidi_workaround': opts.bidi_workaround,
 830         'debug_printtraffic': opts.debug_printtraffic,
 831         'prefer_ffmpeg': opts.prefer_ffmpeg,
 832         'include_ads': opts.include_ads,
 833         'default_search': opts.default_search,
 834         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 835         'encoding': opts.encoding,
 836         'exec_cmd': opts.exec_cmd,
 837     }
 838
 839     with YoutubeDL(ydl_opts) as ydl:
 840         ydl.print_debug_header()
 841         ydl.add_default_info_extractors()
 842
 843         # PostProcessors
 844         # Add the metadata pp first, the other pps will copy it
 845         if opts.addmetadata:
 846             ydl.add_post_processor(FFmpegMetadataPP())
 847         if opts.extractaudio:
 848             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 849         if opts.recodevideo:
 850             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 851         if opts.embedsubtitles:
 852             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 853         if opts.xattrs:
 854             ydl.add_post_processor(XAttrMetadataPP())
 855         if opts.embedthumbnail:
 856             if not opts.addmetadata:
 857                 ydl.add_post_processor(FFmpegAudioFixPP())
 858             ydl.add_post_processor(AtomicParsleyPP())
 859
 860
 861         # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
 862         # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
 863         if opts.exec_cmd:
 864             ydl.add_post_processor(ExecAfterDownloadPP(
 865                 verboseOutput=opts.verbose, exec_cmd=opts.exec_cmd))
 866
 867         # Update version
 868         if opts.update_self:
 869             update_self(ydl.to_screen, opts.verbose)
 870
 871         # Remove cache dir
 872         if opts.rm_cachedir:
 873             ydl.cache.remove()
 874
 875         # Maybe do nothing
 876         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 877             if not (opts.update_self or opts.rm_cachedir):
 878                 parser.error(u'you must provide at least one URL')
 879             else:
 880                 sys.exit()
 881
 882         try:
 883             if opts.load_info_filename is not None:
 884                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 885             else:
 886                 retcode = ydl.download(all_urls)
 887         except MaxDownloadsReached:
 888             ydl.to_screen(u'--max-download limit reached, aborting.')
 889             retcode = 101
 890
 891     sys.exit(retcode)
 892
 893
 894 def main(argv=None):
 895     try:
 896         _real_main(argv)
 897     except DownloadError:
 898         sys.exit(1)
 899     except SameFileError:
 900         sys.exit(u'ERROR: fixed output name but more than one file to download')
 901     except KeyboardInterrupt:
 902         sys.exit(u'\nERROR: Interrupted by user')