youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60 )
  61
  62 __license__ = 'Public Domain'
  63
  64 import codecs
  65 import io
  66 import locale
  67 import optparse
  68 import os
  69 import random
  70 import re
  71 import shlex
  72 import sys
  73
  74
  75 from .utils import (
  76     compat_getpass,
  77     compat_print,
  78     DateRange,
  79     DEFAULT_OUTTMPL,
  80     decodeOption,
  81     get_term_width,
  82     DownloadError,
  83     get_cachedir,
  84     MaxDownloadsReached,
  85     preferredencoding,
  86     read_batch_urls,
  87     SameFileError,
  88     setproctitle,
  89     std_headers,
  90     write_string,
  91 )
  92 from .update import update_self
  93 from .FileDownloader import (
  94     FileDownloader,
  95 )
  96 from .extractor import gen_extractors
  97 from .version import __version__
  98 from .YoutubeDL import YoutubeDL
  99 from .postprocessor import (
 100     AtomicParsleyPP,
 101     FFmpegAudioFixPP,
 102     FFmpegMetadataPP,
 103     FFmpegVideoConvertor,
 104     FFmpegExtractAudioPP,
 105     FFmpegEmbedSubtitlePP,
 106     XAttrMetadataPP,
 107 )
 108
 109
 110 def parseOpts(overrideArguments=None):
 111     def _readOptions(filename_bytes, default=[]):
 112         try:
 113             optionf = open(filename_bytes)
 114         except IOError:
 115             return default  # silently skip if file is not present
 116         try:
 117             res = []
 118             for l in optionf:
 119                 res += shlex.split(l, comments=True)
 120         finally:
 121             optionf.close()
 122         return res
 123
 124     def _readUserConf():
 125         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 126         if xdg_config_home:
 127             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 128             if not os.path.isfile(userConfFile):
 129                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 130         else:
 131             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 132             if not os.path.isfile(userConfFile):
 133                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 134         userConf = _readOptions(userConfFile, None)
 135
 136         if userConf is None:
 137             appdata_dir = os.environ.get('appdata')
 138             if appdata_dir:
 139                 userConf = _readOptions(
 140                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 141                     default=None)
 142                 if userConf is None:
 143                     userConf = _readOptions(
 144                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 145                         default=None)
 146
 147         if userConf is None:
 148             userConf = _readOptions(
 149                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 150                 default=None)
 151         if userConf is None:
 152             userConf = _readOptions(
 153                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 154                 default=None)
 155
 156         if userConf is None:
 157             userConf = []
 158
 159         return userConf
 160
 161     def _format_option_string(option):
 162         ''' ('-o', '--option') -> -o, --format METAVAR'''
 163
 164         opts = []
 165
 166         if option._short_opts:
 167             opts.append(option._short_opts[0])
 168         if option._long_opts:
 169             opts.append(option._long_opts[0])
 170         if len(opts) > 1:
 171             opts.insert(1, ', ')
 172
 173         if option.takes_value(): opts.append(' %s' % option.metavar)
 174
 175         return "".join(opts)
 176
 177     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 178         setattr(parser.values, option.dest, value.split(','))
 179
 180     def _hide_login_info(opts):
 181         opts = list(opts)
 182         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 183             try:
 184                 i = opts.index(private_opt)
 185                 opts[i+1] = '<PRIVATE>'
 186             except ValueError:
 187                 pass
 188         return opts
 189
 190     max_width = 80
 191     max_help_position = 80
 192
 193     # No need to wrap help messages if we're on a wide console
 194     columns = get_term_width()
 195     if columns: max_width = columns
 196
 197     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 198     fmt.format_option_strings = _format_option_string
 199
 200     kw = {
 201         'version'   : __version__,
 202         'formatter' : fmt,
 203         'usage' : '%prog [options] url [url...]',
 204         'conflict_handler' : 'resolve',
 205     }
 206
 207     parser = optparse.OptionParser(**kw)
 208
 209     # option groups
 210     general        = optparse.OptionGroup(parser, 'General Options')
 211     selection      = optparse.OptionGroup(parser, 'Video Selection')
 212     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 213     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 214     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 215     downloader     = optparse.OptionGroup(parser, 'Download Options')
 216     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 217     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 218     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 219
 220     general.add_option('-h', '--help',
 221             action='help', help='print this help text and exit')
 222     general.add_option('-v', '--version',
 223             action='version', help='print program version and exit')
 224     general.add_option('-U', '--update',
 225             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 226     general.add_option('-i', '--ignore-errors',
 227             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 228     general.add_option('--abort-on-error',
 229             action='store_false', dest='ignoreerrors',
 230             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 231     general.add_option('--dump-user-agent',
 232             action='store_true', dest='dump_user_agent',
 233             help='display the current browser identification', default=False)
 234     general.add_option('--user-agent',
 235             dest='user_agent', help='specify a custom user agent', metavar='UA')
 236     general.add_option('--referer',
 237             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 238             metavar='REF', default=None)
 239     general.add_option('--add-header',
 240             dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append",
 241             metavar='FIELD:VALUE')
 242     general.add_option('--list-extractors',
 243             action='store_true', dest='list_extractors',
 244             help='List all supported extractors and the URLs they would handle', default=False)
 245     general.add_option('--extractor-descriptions',
 246             action='store_true', dest='list_extractor_descriptions',
 247             help='Output descriptions of all supported extractors', default=False)
 248     general.add_option(
 249         '--proxy', dest='proxy', default=None, metavar='URL',
 250         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 251     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 252     general.add_option(
 253         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 254         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 255     general.add_option(
 256         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 257         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 258     general.add_option(
 259         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 260         help='Disable filesystem caching')
 261     general.add_option(
 262         '--socket-timeout', dest='socket_timeout',
 263         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 264     general.add_option(
 265         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 266         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 267     general.add_option(
 268         '--default-search',
 269         dest='default_search', metavar='PREFIX',
 270         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". By default (with value "auto") youtube-dl guesses.')
 271     general.add_option(
 272         '--ignore-config',
 273         action='store_true',
 274         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 275     general.add_option(
 276         '--encoding', dest='encoding', metavar='ENCODING',
 277         help='Force the specified encoding (experimental)')
 278
 279     selection.add_option(
 280         '--playlist-start',
 281         dest='playliststart', metavar='NUMBER', default=1, type=int,
 282         help='playlist video to start at (default is %default)')
 283     selection.add_option(
 284         '--playlist-end',
 285         dest='playlistend', metavar='NUMBER', default=None, type=int,
 286         help='playlist video to end at (default is last)')
 287     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 288     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 289     selection.add_option('--max-downloads', metavar='NUMBER',
 290                          dest='max_downloads', type=int, default=None,
 291                          help='Abort after downloading NUMBER files')
 292     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 293     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 294     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 295     selection.add_option(
 296         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 297         help='download only videos uploaded on or before this date (i.e. inclusive)')
 298     selection.add_option(
 299         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 300         help='download only videos uploaded on or after this date (i.e. inclusive)')
 301     selection.add_option(
 302         '--min-views', metavar='COUNT', dest='min_views',
 303         default=None, type=int,
 304         help="Do not download any videos with less than COUNT views",)
 305     selection.add_option(
 306         '--max-views', metavar='COUNT', dest='max_views',
 307         default=None, type=int,
 308         help="Do not download any videos with more than COUNT views",)
 309     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 310     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 311                          help='download only videos suitable for the given age',
 312                          default=None, type=int)
 313     selection.add_option('--download-archive', metavar='FILE',
 314                          dest='download_archive',
 315                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 316     selection.add_option(
 317         '--include-ads', dest='include_ads',
 318         action='store_true',
 319         help='Download advertisements as well (experimental)')
 320     selection.add_option(
 321         '--youtube-include-dash-manifest', action='store_true',
 322         dest='youtube_include_dash_manifest', default=False,
 323         help='Try to download the DASH manifest on YouTube videos (experimental)')
 324
 325     authentication.add_option('-u', '--username',
 326             dest='username', metavar='USERNAME', help='account username')
 327     authentication.add_option('-p', '--password',
 328             dest='password', metavar='PASSWORD', help='account password')
 329     authentication.add_option('-n', '--netrc',
 330             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 331     authentication.add_option('--video-password',
 332             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 333
 334
 335     video_format.add_option('-f', '--format',
 336             action='store', dest='format', metavar='FORMAT', default=None,
 337             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 338     video_format.add_option('--all-formats',
 339             action='store_const', dest='format', help='download all available video formats', const='all')
 340     video_format.add_option('--prefer-free-formats',
 341             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 342     video_format.add_option('--max-quality',
 343             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 344     video_format.add_option('-F', '--list-formats',
 345             action='store_true', dest='listformats', help='list all available formats')
 346
 347     subtitles.add_option('--write-sub', '--write-srt',
 348             action='store_true', dest='writesubtitles',
 349             help='write subtitle file', default=False)
 350     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 351             action='store_true', dest='writeautomaticsub',
 352             help='write automatic subtitle file (youtube only)', default=False)
 353     subtitles.add_option('--all-subs',
 354             action='store_true', dest='allsubtitles',
 355             help='downloads all the available subtitles of the video', default=False)
 356     subtitles.add_option('--list-subs',
 357             action='store_true', dest='listsubtitles',
 358             help='lists all available subtitles for the video', default=False)
 359     subtitles.add_option('--sub-format',
 360             action='store', dest='subtitlesformat', metavar='FORMAT',
 361             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 362     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 363             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 364             default=[], callback=_comma_separated_values_options_callback,
 365             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 366
 367     downloader.add_option('-r', '--rate-limit',
 368             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 369     downloader.add_option('-R', '--retries',
 370             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 371     downloader.add_option('--buffer-size',
 372             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 373     downloader.add_option('--no-resize-buffer',
 374             action='store_true', dest='noresizebuffer',
 375             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 376     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 377
 378     verbosity.add_option('-q', '--quiet',
 379             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 380     verbosity.add_option(
 381         '--no-warnings',
 382         dest='no_warnings', action='store_true', default=False,
 383         help='Ignore warnings')
 384     verbosity.add_option('-s', '--simulate',
 385             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 386     verbosity.add_option('--skip-download',
 387             action='store_true', dest='skip_download', help='do not download the video', default=False)
 388     verbosity.add_option('-g', '--get-url',
 389             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 390     verbosity.add_option('-e', '--get-title',
 391             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 392     verbosity.add_option('--get-id',
 393             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 394     verbosity.add_option('--get-thumbnail',
 395             action='store_true', dest='getthumbnail',
 396             help='simulate, quiet but print thumbnail URL', default=False)
 397     verbosity.add_option('--get-description',
 398             action='store_true', dest='getdescription',
 399             help='simulate, quiet but print video description', default=False)
 400     verbosity.add_option('--get-duration',
 401             action='store_true', dest='getduration',
 402             help='simulate, quiet but print video length', default=False)
 403     verbosity.add_option('--get-filename',
 404             action='store_true', dest='getfilename',
 405             help='simulate, quiet but print output filename', default=False)
 406     verbosity.add_option('--get-format',
 407             action='store_true', dest='getformat',
 408             help='simulate, quiet but print output format', default=False)
 409     verbosity.add_option('-j', '--dump-json',
 410             action='store_true', dest='dumpjson',
 411             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 412     verbosity.add_option('--newline',
 413             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 414     verbosity.add_option('--no-progress',
 415             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 416     verbosity.add_option('--console-title',
 417             action='store_true', dest='consoletitle',
 418             help='display progress in console titlebar', default=False)
 419     verbosity.add_option('-v', '--verbose',
 420             action='store_true', dest='verbose', help='print various debugging information', default=False)
 421     verbosity.add_option('--dump-intermediate-pages',
 422             action='store_true', dest='dump_intermediate_pages', default=False,
 423             help='print downloaded pages to debug problems (very verbose)')
 424     verbosity.add_option('--write-pages',
 425             action='store_true', dest='write_pages', default=False,
 426             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 427     verbosity.add_option('--youtube-print-sig-code',
 428             action='store_true', dest='youtube_print_sig_code', default=False,
 429             help=optparse.SUPPRESS_HELP)
 430     verbosity.add_option('--print-traffic',
 431             dest='debug_printtraffic', action='store_true', default=False,
 432             help='Display sent and read HTTP traffic')
 433
 434
 435     filesystem.add_option('-t', '--title',
 436             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 437     filesystem.add_option('--id',
 438             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 439     filesystem.add_option('-l', '--literal',
 440             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 441     filesystem.add_option('-A', '--auto-number',
 442             action='store_true', dest='autonumber',
 443             help='number downloaded files starting from 00000', default=False)
 444     filesystem.add_option('-o', '--output',
 445             dest='outtmpl', metavar='TEMPLATE',
 446             help=('output filename template. Use %(title)s to get the title, '
 447                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 448                   '%(autonumber)s to get an automatically incremented number, '
 449                   '%(ext)s for the filename extension, '
 450                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 451                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 452                   '%(upload_date)s for the upload date (YYYYMMDD), '
 453                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 454                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 455                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 456                   '%(height)s and %(width)s for the width and height of the video format. '
 457                   '%(resolution)s for a textual description of the resolution of the video format. '
 458                   'Use - to output to stdout. Can also be used to download to a different directory, '
 459                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 460     filesystem.add_option('--autonumber-size',
 461             dest='autonumber_size', metavar='NUMBER',
 462             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 463     filesystem.add_option('--restrict-filenames',
 464             action='store_true', dest='restrictfilenames',
 465             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 466     filesystem.add_option('-a', '--batch-file',
 467             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 468     filesystem.add_option('--load-info',
 469             dest='load_info_filename', metavar='FILE',
 470             help='json file containing the video information (created with the "--write-json" option)')
 471     filesystem.add_option('-w', '--no-overwrites',
 472             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 473     filesystem.add_option('-c', '--continue',
 474             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 475     filesystem.add_option('--no-continue',
 476             action='store_false', dest='continue_dl',
 477             help='do not resume partially downloaded files (restart from beginning)')
 478     filesystem.add_option('--cookies',
 479             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 480     filesystem.add_option('--no-part',
 481             action='store_true', dest='nopart', help='do not use .part files', default=False)
 482     filesystem.add_option('--no-mtime',
 483             action='store_false', dest='updatetime',
 484             help='do not use the Last-modified header to set the file modification time', default=True)
 485     filesystem.add_option('--write-description',
 486             action='store_true', dest='writedescription',
 487             help='write video description to a .description file', default=False)
 488     filesystem.add_option('--write-info-json',
 489             action='store_true', dest='writeinfojson',
 490             help='write video metadata to a .info.json file', default=False)
 491     filesystem.add_option('--write-annotations',
 492             action='store_true', dest='writeannotations',
 493             help='write video annotations to a .annotation file', default=False)
 494     filesystem.add_option('--write-thumbnail',
 495             action='store_true', dest='writethumbnail',
 496             help='write thumbnail image to disk', default=False)
 497
 498
 499     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 500             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 501     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 502             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 503     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 504             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 505     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 506             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 507     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 508             help='keeps the video file on disk after the post-processing; the video is erased by default')
 509     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 510             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 511     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 512             help='embed subtitles in the video (only for mp4 videos)')
 513     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 514             help='embed thumbnail in the audio as cover art')
 515     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 516             help='write metadata to the video file')
 517     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 518             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 519     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 520         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 521     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 522         help='Prefer ffmpeg over avconv for running the postprocessors')
 523
 524
 525     parser.add_option_group(general)
 526     parser.add_option_group(selection)
 527     parser.add_option_group(downloader)
 528     parser.add_option_group(filesystem)
 529     parser.add_option_group(verbosity)
 530     parser.add_option_group(video_format)
 531     parser.add_option_group(subtitles)
 532     parser.add_option_group(authentication)
 533     parser.add_option_group(postproc)
 534
 535     if overrideArguments is not None:
 536         opts, args = parser.parse_args(overrideArguments)
 537         if opts.verbose:
 538             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 539     else:
 540         commandLineConf = sys.argv[1:]
 541         if '--ignore-config' in commandLineConf:
 542             systemConf = []
 543             userConf = []
 544         else:
 545             systemConf = _readOptions('/etc/youtube-dl.conf')
 546             if '--ignore-config' in systemConf:
 547                 userConf = []
 548             else:
 549                 userConf = _readUserConf()
 550         argv = systemConf + userConf + commandLineConf
 551
 552         opts, args = parser.parse_args(argv)
 553         if opts.verbose:
 554             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 555             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 556             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 557
 558     return parser, opts, args
 559
 560
 561 def _real_main(argv=None):
 562     # Compatibility fixes for Windows
 563     if sys.platform == 'win32':
 564         # https://github.com/rg3/youtube-dl/issues/820
 565         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 566
 567     setproctitle(u'youtube-dl')
 568
 569     parser, opts, args = parseOpts(argv)
 570
 571     # Set user agent
 572     if opts.user_agent is not None:
 573         std_headers['User-Agent'] = opts.user_agent
 574
 575     # Set referer
 576     if opts.referer is not None:
 577         std_headers['Referer'] = opts.referer
 578
 579     # Custom HTTP headers
 580     if opts.headers is not None:
 581         for h in opts.headers:
 582             if h.find(':', 1) < 0:
 583                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 584             key, value = h.split(':', 2)
 585             if opts.verbose:
 586                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 587             std_headers[key] = value
 588
 589     # Dump user agent
 590     if opts.dump_user_agent:
 591         compat_print(std_headers['User-Agent'])
 592         sys.exit(0)
 593
 594     # Batch file verification
 595     batch_urls = []
 596     if opts.batchfile is not None:
 597         try:
 598             if opts.batchfile == '-':
 599                 batchfd = sys.stdin
 600             else:
 601                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 602             batch_urls = read_batch_urls(batchfd)
 603             if opts.verbose:
 604                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 605         except IOError:
 606             sys.exit(u'ERROR: batch file could not be read')
 607     all_urls = batch_urls + args
 608     all_urls = [url.strip() for url in all_urls]
 609     _enc = preferredencoding()
 610     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 611
 612     extractors = gen_extractors()
 613
 614     if opts.list_extractors:
 615         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 616             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 617             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 618             for mu in matchedUrls:
 619                 compat_print(u'  ' + mu)
 620         sys.exit(0)
 621     if opts.list_extractor_descriptions:
 622         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 623             if not ie._WORKING:
 624                 continue
 625             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 626             if desc is False:
 627                 continue
 628             if hasattr(ie, 'SEARCH_KEY'):
 629                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 630                 _COUNTS = (u'', u'5', u'10', u'all')
 631                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 632             compat_print(desc)
 633         sys.exit(0)
 634
 635
 636     # Conflicting, missing and erroneous options
 637     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 638         parser.error(u'using .netrc conflicts with giving username/password')
 639     if opts.password is not None and opts.username is None:
 640         parser.error(u'account username missing\n')
 641     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 642         parser.error(u'using output template conflicts with using title, video ID or auto number')
 643     if opts.usetitle and opts.useid:
 644         parser.error(u'using title conflicts with using video ID')
 645     if opts.username is not None and opts.password is None:
 646         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 647     if opts.ratelimit is not None:
 648         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 649         if numeric_limit is None:
 650             parser.error(u'invalid rate limit specified')
 651         opts.ratelimit = numeric_limit
 652     if opts.min_filesize is not None:
 653         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 654         if numeric_limit is None:
 655             parser.error(u'invalid min_filesize specified')
 656         opts.min_filesize = numeric_limit
 657     if opts.max_filesize is not None:
 658         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 659         if numeric_limit is None:
 660             parser.error(u'invalid max_filesize specified')
 661         opts.max_filesize = numeric_limit
 662     if opts.retries is not None:
 663         try:
 664             opts.retries = int(opts.retries)
 665         except (TypeError, ValueError):
 666             parser.error(u'invalid retry count specified')
 667     if opts.buffersize is not None:
 668         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 669         if numeric_buffersize is None:
 670             parser.error(u'invalid buffer size specified')
 671         opts.buffersize = numeric_buffersize
 672     if opts.playliststart <= 0:
 673         raise ValueError(u'Playlist start must be positive')
 674     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 675         raise ValueError(u'Playlist end must be greater than playlist start')
 676     if opts.extractaudio:
 677         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 678             parser.error(u'invalid audio format specified')
 679     if opts.audioquality:
 680         opts.audioquality = opts.audioquality.strip('k').strip('K')
 681         if not opts.audioquality.isdigit():
 682             parser.error(u'invalid audio quality specified')
 683     if opts.recodevideo is not None:
 684         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 685             parser.error(u'invalid video recode format specified')
 686     if opts.date is not None:
 687         date = DateRange.day(opts.date)
 688     else:
 689         date = DateRange(opts.dateafter, opts.datebefore)
 690     if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
 691         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 692
 693     # Do not download videos when there are audio-only formats
 694     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 695         opts.format = 'bestaudio/best'
 696
 697     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 698     # this was the old behaviour if only --all-sub was given.
 699     if opts.allsubtitles and (opts.writeautomaticsub == False):
 700         opts.writesubtitles = True
 701
 702     if sys.version_info < (3,):
 703         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 704         if opts.outtmpl is not None:
 705             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 706     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 707             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 708             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 709             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 710             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 711             or (opts.useid and u'%(id)s.%(ext)s')
 712             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 713             or DEFAULT_OUTTMPL)
 714     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 715         parser.error(u'Cannot download a video and extract audio into the same'
 716                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 717                      u' template'.format(outtmpl))
 718
 719     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 720     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 721
 722     ydl_opts = {
 723         'usenetrc': opts.usenetrc,
 724         'username': opts.username,
 725         'password': opts.password,
 726         'videopassword': opts.videopassword,
 727         'quiet': (opts.quiet or any_printing),
 728         'no_warnings': opts.no_warnings,
 729         'forceurl': opts.geturl,
 730         'forcetitle': opts.gettitle,
 731         'forceid': opts.getid,
 732         'forcethumbnail': opts.getthumbnail,
 733         'forcedescription': opts.getdescription,
 734         'forceduration': opts.getduration,
 735         'forcefilename': opts.getfilename,
 736         'forceformat': opts.getformat,
 737         'forcejson': opts.dumpjson,
 738         'simulate': opts.simulate,
 739         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 740         'format': opts.format,
 741         'format_limit': opts.format_limit,
 742         'listformats': opts.listformats,
 743         'outtmpl': outtmpl,
 744         'autonumber_size': opts.autonumber_size,
 745         'restrictfilenames': opts.restrictfilenames,
 746         'ignoreerrors': opts.ignoreerrors,
 747         'ratelimit': opts.ratelimit,
 748         'nooverwrites': opts.nooverwrites,
 749         'retries': opts.retries,
 750         'buffersize': opts.buffersize,
 751         'noresizebuffer': opts.noresizebuffer,
 752         'continuedl': opts.continue_dl,
 753         'noprogress': opts.noprogress,
 754         'progress_with_newline': opts.progress_with_newline,
 755         'playliststart': opts.playliststart,
 756         'playlistend': opts.playlistend,
 757         'noplaylist': opts.noplaylist,
 758         'logtostderr': opts.outtmpl == '-',
 759         'consoletitle': opts.consoletitle,
 760         'nopart': opts.nopart,
 761         'updatetime': opts.updatetime,
 762         'writedescription': opts.writedescription,
 763         'writeannotations': opts.writeannotations,
 764         'writeinfojson': opts.writeinfojson,
 765         'writethumbnail': opts.writethumbnail,
 766         'writesubtitles': opts.writesubtitles,
 767         'writeautomaticsub': opts.writeautomaticsub,
 768         'allsubtitles': opts.allsubtitles,
 769         'listsubtitles': opts.listsubtitles,
 770         'subtitlesformat': opts.subtitlesformat,
 771         'subtitleslangs': opts.subtitleslangs,
 772         'matchtitle': decodeOption(opts.matchtitle),
 773         'rejecttitle': decodeOption(opts.rejecttitle),
 774         'max_downloads': opts.max_downloads,
 775         'prefer_free_formats': opts.prefer_free_formats,
 776         'verbose': opts.verbose,
 777         'dump_intermediate_pages': opts.dump_intermediate_pages,
 778         'write_pages': opts.write_pages,
 779         'test': opts.test,
 780         'keepvideo': opts.keepvideo,
 781         'min_filesize': opts.min_filesize,
 782         'max_filesize': opts.max_filesize,
 783         'min_views': opts.min_views,
 784         'max_views': opts.max_views,
 785         'daterange': date,
 786         'cachedir': opts.cachedir,
 787         'youtube_print_sig_code': opts.youtube_print_sig_code,
 788         'age_limit': opts.age_limit,
 789         'download_archive': download_archive_fn,
 790         'cookiefile': opts.cookiefile,
 791         'nocheckcertificate': opts.no_check_certificate,
 792         'prefer_insecure': opts.prefer_insecure,
 793         'proxy': opts.proxy,
 794         'socket_timeout': opts.socket_timeout,
 795         'bidi_workaround': opts.bidi_workaround,
 796         'debug_printtraffic': opts.debug_printtraffic,
 797         'prefer_ffmpeg': opts.prefer_ffmpeg,
 798         'include_ads': opts.include_ads,
 799         'default_search': opts.default_search,
 800         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 801         'encoding': opts.encoding,
 802     }
 803
 804     with YoutubeDL(ydl_opts) as ydl:
 805         ydl.print_debug_header()
 806         ydl.add_default_info_extractors()
 807
 808         # PostProcessors
 809         # Add the metadata pp first, the other pps will copy it
 810         if opts.addmetadata:
 811             ydl.add_post_processor(FFmpegMetadataPP())
 812         if opts.extractaudio:
 813             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 814         if opts.recodevideo:
 815             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 816         if opts.embedsubtitles:
 817             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 818         if opts.xattrs:
 819             ydl.add_post_processor(XAttrMetadataPP())
 820         if opts.embedthumbnail:
 821             if not opts.addmetadata:
 822                 ydl.add_post_processor(FFmpegAudioFixPP())
 823             ydl.add_post_processor(AtomicParsleyPP())
 824
 825         # Update version
 826         if opts.update_self:
 827             update_self(ydl.to_screen, opts.verbose)
 828
 829         # Maybe do nothing
 830         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 831             if not opts.update_self:
 832                 parser.error(u'you must provide at least one URL')
 833             else:
 834                 sys.exit()
 835
 836         try:
 837             if opts.load_info_filename is not None:
 838                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 839             else:
 840                 retcode = ydl.download(all_urls)
 841         except MaxDownloadsReached:
 842             ydl.to_screen(u'--max-download limit reached, aborting.')
 843             retcode = 101
 844
 845     sys.exit(retcode)
 846
 847
 848 def main(argv=None):
 849     try:
 850         _real_main(argv)
 851     except DownloadError:
 852         sys.exit(1)
 853     except SameFileError:
 854         sys.exit(u'ERROR: fixed output name but more than one file to download')
 855     except KeyboardInterrupt:
 856         sys.exit(u'\nERROR: Interrupted by user')