youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70     'David Fabijan',
  71     'Sebastian Haas',
  72 )
  73
  74 __license__ = 'Public Domain'
  75
  76 import codecs
  77 import io
  78 import optparse
  79 import os
  80 import random
  81 import shlex
  82 import shutil
  83 import sys
  84
  85
  86 from .utils import (
  87     compat_getpass,
  88     compat_print,
  89     DateRange,
  90     DEFAULT_OUTTMPL,
  91     decodeOption,
  92     get_term_width,
  93     DownloadError,
  94     get_cachedir,
  95     MaxDownloadsReached,
  96     preferredencoding,
  97     read_batch_urls,
  98     SameFileError,
  99     setproctitle,
 100     std_headers,
 101     write_string,
 102 )
 103 from .update import update_self
 104 from .downloader import (
 105     FileDownloader,
 106 )
 107 from .extractor import gen_extractors
 108 from .version import __version__
 109 from .YoutubeDL import YoutubeDL
 110 from .postprocessor import (
 111     AtomicParsleyPP,
 112     FFmpegAudioFixPP,
 113     FFmpegMetadataPP,
 114     FFmpegVideoConvertor,
 115     FFmpegExtractAudioPP,
 116     FFmpegEmbedSubtitlePP,
 117     XAttrMetadataPP,
 118 )
 119
 120
 121 def parseOpts(overrideArguments=None):
 122     def _readOptions(filename_bytes, default=[]):
 123         try:
 124             optionf = open(filename_bytes)
 125         except IOError:
 126             return default  # silently skip if file is not present
 127         try:
 128             res = []
 129             for l in optionf:
 130                 res += shlex.split(l, comments=True)
 131         finally:
 132             optionf.close()
 133         return res
 134
 135     def _readUserConf():
 136         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 137         if xdg_config_home:
 138             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 139             if not os.path.isfile(userConfFile):
 140                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 141         else:
 142             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 143             if not os.path.isfile(userConfFile):
 144                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 145         userConf = _readOptions(userConfFile, None)
 146
 147         if userConf is None:
 148             appdata_dir = os.environ.get('appdata')
 149             if appdata_dir:
 150                 userConf = _readOptions(
 151                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 152                     default=None)
 153                 if userConf is None:
 154                     userConf = _readOptions(
 155                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 156                         default=None)
 157
 158         if userConf is None:
 159             userConf = _readOptions(
 160                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 161                 default=None)
 162         if userConf is None:
 163             userConf = _readOptions(
 164                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 165                 default=None)
 166
 167         if userConf is None:
 168             userConf = []
 169
 170         return userConf
 171
 172     def _format_option_string(option):
 173         ''' ('-o', '--option') -> -o, --format METAVAR'''
 174
 175         opts = []
 176
 177         if option._short_opts:
 178             opts.append(option._short_opts[0])
 179         if option._long_opts:
 180             opts.append(option._long_opts[0])
 181         if len(opts) > 1:
 182             opts.insert(1, ', ')
 183
 184         if option.takes_value(): opts.append(' %s' % option.metavar)
 185
 186         return "".join(opts)
 187
 188     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 189         setattr(parser.values, option.dest, value.split(','))
 190
 191     def _hide_login_info(opts):
 192         opts = list(opts)
 193         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 194             try:
 195                 i = opts.index(private_opt)
 196                 opts[i+1] = '<PRIVATE>'
 197             except ValueError:
 198                 pass
 199         return opts
 200
 201     max_width = 80
 202     max_help_position = 80
 203
 204     # No need to wrap help messages if we're on a wide console
 205     columns = get_term_width()
 206     if columns: max_width = columns
 207
 208     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 209     fmt.format_option_strings = _format_option_string
 210
 211     kw = {
 212         'version'   : __version__,
 213         'formatter' : fmt,
 214         'usage' : '%prog [options] url [url...]',
 215         'conflict_handler' : 'resolve',
 216     }
 217
 218     parser = optparse.OptionParser(**kw)
 219
 220     # option groups
 221     general        = optparse.OptionGroup(parser, 'General Options')
 222     selection      = optparse.OptionGroup(parser, 'Video Selection')
 223     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 224     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 225     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 226     downloader     = optparse.OptionGroup(parser, 'Download Options')
 227     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 228     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 229     workarounds    = optparse.OptionGroup(parser, 'Workarounds')
 230     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 231
 232     general.add_option('-h', '--help',
 233             action='help', help='print this help text and exit')
 234     general.add_option('-v', '--version',
 235             action='version', help='print program version and exit')
 236     general.add_option('-U', '--update',
 237             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 238     general.add_option('-i', '--ignore-errors',
 239             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 240     general.add_option('--abort-on-error',
 241             action='store_false', dest='ignoreerrors',
 242             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 243     general.add_option('--dump-user-agent',
 244             action='store_true', dest='dump_user_agent',
 245             help='display the current browser identification', default=False)
 246     general.add_option('--list-extractors',
 247             action='store_true', dest='list_extractors',
 248             help='List all supported extractors and the URLs they would handle', default=False)
 249     general.add_option('--extractor-descriptions',
 250             action='store_true', dest='list_extractor_descriptions',
 251             help='Output descriptions of all supported extractors', default=False)
 252     general.add_option(
 253         '--proxy', dest='proxy', default=None, metavar='URL',
 254         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 255     general.add_option(
 256         '--socket-timeout', dest='socket_timeout',
 257         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 258     general.add_option(
 259         '--default-search',
 260         dest='default_search', metavar='PREFIX',
 261         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
 262     general.add_option(
 263         '--ignore-config',
 264         action='store_true',
 265         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 266
 267     selection.add_option(
 268         '--playlist-start',
 269         dest='playliststart', metavar='NUMBER', default=1, type=int,
 270         help='playlist video to start at (default is %default)')
 271     selection.add_option(
 272         '--playlist-end',
 273         dest='playlistend', metavar='NUMBER', default=None, type=int,
 274         help='playlist video to end at (default is last)')
 275     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 276     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 277     selection.add_option('--max-downloads', metavar='NUMBER',
 278                          dest='max_downloads', type=int, default=None,
 279                          help='Abort after downloading NUMBER files')
 280     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 281     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 282     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 283     selection.add_option(
 284         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 285         help='download only videos uploaded on or before this date (i.e. inclusive)')
 286     selection.add_option(
 287         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 288         help='download only videos uploaded on or after this date (i.e. inclusive)')
 289     selection.add_option(
 290         '--min-views', metavar='COUNT', dest='min_views',
 291         default=None, type=int,
 292         help="Do not download any videos with less than COUNT views",)
 293     selection.add_option(
 294         '--max-views', metavar='COUNT', dest='max_views',
 295         default=None, type=int,
 296         help="Do not download any videos with more than COUNT views",)
 297     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 298     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 299                          help='download only videos suitable for the given age',
 300                          default=None, type=int)
 301     selection.add_option('--download-archive', metavar='FILE',
 302                          dest='download_archive',
 303                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 304     selection.add_option(
 305         '--include-ads', dest='include_ads',
 306         action='store_true',
 307         help='Download advertisements as well (experimental)')
 308     selection.add_option(
 309         '--youtube-include-dash-manifest', action='store_true',
 310         dest='youtube_include_dash_manifest', default=False,
 311         help='Try to download the DASH manifest on YouTube videos (experimental)')
 312
 313     authentication.add_option('-u', '--username',
 314             dest='username', metavar='USERNAME', help='account username')
 315     authentication.add_option('-p', '--password',
 316             dest='password', metavar='PASSWORD', help='account password')
 317     authentication.add_option('-n', '--netrc',
 318             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 319     authentication.add_option('--video-password',
 320             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 321
 322
 323     video_format.add_option('-f', '--format',
 324             action='store', dest='format', metavar='FORMAT', default=None,
 325             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 326     video_format.add_option('--all-formats',
 327             action='store_const', dest='format', help='download all available video formats', const='all')
 328     video_format.add_option('--prefer-free-formats',
 329             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 330     video_format.add_option('--max-quality',
 331             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 332     video_format.add_option('-F', '--list-formats',
 333             action='store_true', dest='listformats', help='list all available formats')
 334
 335     subtitles.add_option('--write-sub', '--write-srt',
 336             action='store_true', dest='writesubtitles',
 337             help='write subtitle file', default=False)
 338     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 339             action='store_true', dest='writeautomaticsub',
 340             help='write automatic subtitle file (youtube only)', default=False)
 341     subtitles.add_option('--all-subs',
 342             action='store_true', dest='allsubtitles',
 343             help='downloads all the available subtitles of the video', default=False)
 344     subtitles.add_option('--list-subs',
 345             action='store_true', dest='listsubtitles',
 346             help='lists all available subtitles for the video', default=False)
 347     subtitles.add_option('--sub-format',
 348             action='store', dest='subtitlesformat', metavar='FORMAT',
 349             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 350     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 351             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 352             default=[], callback=_comma_separated_values_options_callback,
 353             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 354
 355     downloader.add_option('-r', '--rate-limit',
 356             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 357     downloader.add_option('--sleep-interval',
 358             dest='sleepinterval', metavar='SLEEPINTERVAL', help='number of seconds to sleep between downloads (default is %default)', default="0")
 359     downloader.add_option('-R', '--retries',
 360             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 361     downloader.add_option('--buffer-size',
 362             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 363     downloader.add_option('--no-resize-buffer',
 364             action='store_true', dest='noresizebuffer',
 365             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 366     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 367
 368     workarounds.add_option(
 369         '--encoding', dest='encoding', metavar='ENCODING',
 370         help='Force the specified encoding (experimental)')
 371     workarounds.add_option(
 372         '--no-check-certificate', action='store_true',
 373         dest='no_check_certificate', default=False,
 374         help='Suppress HTTPS certificate validation.')
 375     workarounds.add_option(
 376         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 377         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 378     workarounds.add_option(
 379         '--user-agent', metavar='UA',
 380         dest='user_agent', help='specify a custom user agent')
 381     workarounds.add_option(
 382         '--referer', metavar='REF',
 383         dest='referer', default=None,
 384         help='specify a custom referer, use if the video access is restricted to one domain',
 385     )
 386     workarounds.add_option(
 387         '--add-header', metavar='FIELD:VALUE',
 388         dest='headers', action='append',
 389         help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
 390     )
 391     workarounds.add_option(
 392         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 393         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 394
 395     verbosity.add_option('-q', '--quiet',
 396             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 397     verbosity.add_option(
 398         '--no-warnings',
 399         dest='no_warnings', action='store_true', default=False,
 400         help='Ignore warnings')
 401     verbosity.add_option('-s', '--simulate',
 402             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 403     verbosity.add_option('--skip-download',
 404             action='store_true', dest='skip_download', help='do not download the video', default=False)
 405     verbosity.add_option('-g', '--get-url',
 406             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 407     verbosity.add_option('-e', '--get-title',
 408             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 409     verbosity.add_option('--get-id',
 410             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 411     verbosity.add_option('--get-thumbnail',
 412             action='store_true', dest='getthumbnail',
 413             help='simulate, quiet but print thumbnail URL', default=False)
 414     verbosity.add_option('--get-description',
 415             action='store_true', dest='getdescription',
 416             help='simulate, quiet but print video description', default=False)
 417     verbosity.add_option('--get-duration',
 418             action='store_true', dest='getduration',
 419             help='simulate, quiet but print video length', default=False)
 420     verbosity.add_option('--get-filename',
 421             action='store_true', dest='getfilename',
 422             help='simulate, quiet but print output filename', default=False)
 423     verbosity.add_option('--get-format',
 424             action='store_true', dest='getformat',
 425             help='simulate, quiet but print output format', default=False)
 426     verbosity.add_option('-j', '--dump-json',
 427             action='store_true', dest='dumpjson',
 428             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 429     verbosity.add_option('--newline',
 430             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 431     verbosity.add_option('--no-progress',
 432             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 433     verbosity.add_option('--console-title',
 434             action='store_true', dest='consoletitle',
 435             help='display progress in console titlebar', default=False)
 436     verbosity.add_option('-v', '--verbose',
 437             action='store_true', dest='verbose', help='print various debugging information', default=False)
 438     verbosity.add_option('--dump-intermediate-pages',
 439             action='store_true', dest='dump_intermediate_pages', default=False,
 440             help='print downloaded pages to debug problems (very verbose)')
 441     verbosity.add_option('--write-pages',
 442             action='store_true', dest='write_pages', default=False,
 443             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 444     verbosity.add_option('--youtube-print-sig-code',
 445             action='store_true', dest='youtube_print_sig_code', default=False,
 446             help=optparse.SUPPRESS_HELP)
 447     verbosity.add_option('--print-traffic',
 448             dest='debug_printtraffic', action='store_true', default=False,
 449             help='Display sent and read HTTP traffic')
 450
 451
 452     filesystem.add_option('-a', '--batch-file',
 453             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 454     filesystem.add_option('--id',
 455             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 456     filesystem.add_option('-A', '--auto-number',
 457             action='store_true', dest='autonumber',
 458             help='number downloaded files starting from 00000', default=False)
 459     filesystem.add_option('-o', '--output',
 460             dest='outtmpl', metavar='TEMPLATE',
 461             help=('output filename template. Use %(title)s to get the title, '
 462                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 463                   '%(autonumber)s to get an automatically incremented number, '
 464                   '%(ext)s for the filename extension, '
 465                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 466                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 467                   '%(upload_date)s for the upload date (YYYYMMDD), '
 468                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 469                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 470                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 471                   '%(height)s and %(width)s for the width and height of the video format. '
 472                   '%(resolution)s for a textual description of the resolution of the video format. '
 473                   'Use - to output to stdout. Can also be used to download to a different directory, '
 474                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 475     filesystem.add_option('--autonumber-size',
 476             dest='autonumber_size', metavar='NUMBER',
 477             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 478     filesystem.add_option('--restrict-filenames',
 479             action='store_true', dest='restrictfilenames',
 480             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 481     filesystem.add_option('-t', '--title',
 482             action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
 483     filesystem.add_option('-l', '--literal',
 484             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 485     filesystem.add_option('-w', '--no-overwrites',
 486             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 487     filesystem.add_option('-c', '--continue',
 488             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 489     filesystem.add_option('--no-continue',
 490             action='store_false', dest='continue_dl',
 491             help='do not resume partially downloaded files (restart from beginning)')
 492     filesystem.add_option('--no-part',
 493             action='store_true', dest='nopart', help='do not use .part files', default=False)
 494     filesystem.add_option('--no-mtime',
 495             action='store_false', dest='updatetime',
 496             help='do not use the Last-modified header to set the file modification time', default=True)
 497     filesystem.add_option('--write-description',
 498             action='store_true', dest='writedescription',
 499             help='write video description to a .description file', default=False)
 500     filesystem.add_option('--write-info-json',
 501             action='store_true', dest='writeinfojson',
 502             help='write video metadata to a .info.json file', default=False)
 503     filesystem.add_option('--write-annotations',
 504             action='store_true', dest='writeannotations',
 505             help='write video annotations to a .annotation file', default=False)
 506     filesystem.add_option('--write-thumbnail',
 507             action='store_true', dest='writethumbnail',
 508             help='write thumbnail image to disk', default=False)
 509     filesystem.add_option('--load-info',
 510             dest='load_info_filename', metavar='FILE',
 511             help='json file containing the video information (created with the "--write-json" option)')
 512     filesystem.add_option('--cookies',
 513             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 514     filesystem.add_option(
 515         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 516         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 517     filesystem.add_option(
 518         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 519         help='Disable filesystem caching')
 520     filesystem.add_option(
 521         '--rm-cache-dir', action='store_true', dest='rm_cachedir',
 522         help='Delete all filesystem cache files')
 523
 524
 525     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 526             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 527     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 528             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 529     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 530             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 531     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 532             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
 533     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 534             help='keeps the video file on disk after the post-processing; the video is erased by default')
 535     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 536             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 537     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 538             help='embed subtitles in the video (only for mp4 videos)')
 539     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 540             help='embed thumbnail in the audio as cover art')
 541     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 542             help='write metadata to the video file')
 543     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 544             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 545     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 546         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 547     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 548         help='Prefer ffmpeg over avconv for running the postprocessors')
 549
 550
 551     parser.add_option_group(general)
 552     parser.add_option_group(selection)
 553     parser.add_option_group(downloader)
 554     parser.add_option_group(filesystem)
 555     parser.add_option_group(verbosity)
 556     parser.add_option_group(workarounds)
 557     parser.add_option_group(video_format)
 558     parser.add_option_group(subtitles)
 559     parser.add_option_group(authentication)
 560     parser.add_option_group(postproc)
 561
 562     if overrideArguments is not None:
 563         opts, args = parser.parse_args(overrideArguments)
 564         if opts.verbose:
 565             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 566     else:
 567         commandLineConf = sys.argv[1:]
 568         if '--ignore-config' in commandLineConf:
 569             systemConf = []
 570             userConf = []
 571         else:
 572             systemConf = _readOptions('/etc/youtube-dl.conf')
 573             if '--ignore-config' in systemConf:
 574                 userConf = []
 575             else:
 576                 userConf = _readUserConf()
 577         argv = systemConf + userConf + commandLineConf
 578
 579         opts, args = parser.parse_args(argv)
 580         if opts.verbose:
 581             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 582             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 583             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 584
 585     return parser, opts, args
 586
 587
 588 def _real_main(argv=None):
 589     # Compatibility fixes for Windows
 590     if sys.platform == 'win32':
 591         # https://github.com/rg3/youtube-dl/issues/820
 592         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 593
 594     setproctitle(u'youtube-dl')
 595
 596     parser, opts, args = parseOpts(argv)
 597
 598     # Set user agent
 599     if opts.user_agent is not None:
 600         std_headers['User-Agent'] = opts.user_agent
 601
 602     # Set referer
 603     if opts.referer is not None:
 604         std_headers['Referer'] = opts.referer
 605
 606     # Custom HTTP headers
 607     if opts.headers is not None:
 608         for h in opts.headers:
 609             if h.find(':', 1) < 0:
 610                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 611             key, value = h.split(':', 2)
 612             if opts.verbose:
 613                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 614             std_headers[key] = value
 615
 616     # Dump user agent
 617     if opts.dump_user_agent:
 618         compat_print(std_headers['User-Agent'])
 619         sys.exit(0)
 620
 621     # Batch file verification
 622     batch_urls = []
 623     if opts.batchfile is not None:
 624         try:
 625             if opts.batchfile == '-':
 626                 batchfd = sys.stdin
 627             else:
 628                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 629             batch_urls = read_batch_urls(batchfd)
 630             if opts.verbose:
 631                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 632         except IOError:
 633             sys.exit(u'ERROR: batch file could not be read')
 634     all_urls = batch_urls + args
 635     all_urls = [url.strip() for url in all_urls]
 636     _enc = preferredencoding()
 637     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 638
 639     extractors = gen_extractors()
 640
 641     if opts.list_extractors:
 642         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 643             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 644             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 645             for mu in matchedUrls:
 646                 compat_print(u'  ' + mu)
 647         sys.exit(0)
 648     if opts.list_extractor_descriptions:
 649         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 650             if not ie._WORKING:
 651                 continue
 652             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 653             if desc is False:
 654                 continue
 655             if hasattr(ie, 'SEARCH_KEY'):
 656                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 657                 _COUNTS = (u'', u'5', u'10', u'all')
 658                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 659             compat_print(desc)
 660         sys.exit(0)
 661
 662
 663     # Conflicting, missing and erroneous options
 664     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 665         parser.error(u'using .netrc conflicts with giving username/password')
 666     if opts.password is not None and opts.username is None:
 667         parser.error(u'account username missing\n')
 668     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 669         parser.error(u'using output template conflicts with using title, video ID or auto number')
 670     if opts.usetitle and opts.useid:
 671         parser.error(u'using title conflicts with using video ID')
 672     if opts.username is not None and opts.password is None:
 673         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 674     if opts.ratelimit is not None:
 675         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 676         if numeric_limit is None:
 677             parser.error(u'invalid rate limit specified')
 678         opts.ratelimit = numeric_limit
 679     if opts.sleepinterval is not None:
 680         try:
 681             opts.sleepinterval = abs(int(opts.sleepinterval))
 682         except ValueError:
 683             parser.error(u'invalid sleep interval specified')
 684     if opts.min_filesize is not None:
 685         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 686         if numeric_limit is None:
 687             parser.error(u'invalid min_filesize specified')
 688         opts.min_filesize = numeric_limit
 689     if opts.max_filesize is not None:
 690         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 691         if numeric_limit is None:
 692             parser.error(u'invalid max_filesize specified')
 693         opts.max_filesize = numeric_limit
 694     if opts.retries is not None:
 695         try:
 696             opts.retries = int(opts.retries)
 697         except (TypeError, ValueError):
 698             parser.error(u'invalid retry count specified')
 699     if opts.buffersize is not None:
 700         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 701         if numeric_buffersize is None:
 702             parser.error(u'invalid buffer size specified')
 703         opts.buffersize = numeric_buffersize
 704     if opts.playliststart <= 0:
 705         raise ValueError(u'Playlist start must be positive')
 706     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 707         raise ValueError(u'Playlist end must be greater than playlist start')
 708     if opts.extractaudio:
 709         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 710             parser.error(u'invalid audio format specified')
 711     if opts.audioquality:
 712         opts.audioquality = opts.audioquality.strip('k').strip('K')
 713         if not opts.audioquality.isdigit():
 714             parser.error(u'invalid audio quality specified')
 715     if opts.recodevideo is not None:
 716         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 717             parser.error(u'invalid video recode format specified')
 718     if opts.date is not None:
 719         date = DateRange.day(opts.date)
 720     else:
 721         date = DateRange(opts.dateafter, opts.datebefore)
 722     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 723         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 724
 725     # Do not download videos when there are audio-only formats
 726     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 727         opts.format = 'bestaudio/best'
 728
 729     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 730     # this was the old behaviour if only --all-sub was given.
 731     if opts.allsubtitles and (opts.writeautomaticsub == False):
 732         opts.writesubtitles = True
 733
 734     if sys.version_info < (3,):
 735         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 736         if opts.outtmpl is not None:
 737             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 738     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 739             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 740             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 741             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 742             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 743             or (opts.useid and u'%(id)s.%(ext)s')
 744             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 745             or DEFAULT_OUTTMPL)
 746     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 747         parser.error(u'Cannot download a video and extract audio into the same'
 748                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 749                      u' template'.format(outtmpl))
 750
 751     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 752     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 753
 754     ydl_opts = {
 755         'usenetrc': opts.usenetrc,
 756         'username': opts.username,
 757         'password': opts.password,
 758         'videopassword': opts.videopassword,
 759         'quiet': (opts.quiet or any_printing),
 760         'no_warnings': opts.no_warnings,
 761         'forceurl': opts.geturl,
 762         'forcetitle': opts.gettitle,
 763         'forceid': opts.getid,
 764         'forcethumbnail': opts.getthumbnail,
 765         'forcedescription': opts.getdescription,
 766         'forceduration': opts.getduration,
 767         'forcefilename': opts.getfilename,
 768         'forceformat': opts.getformat,
 769         'forcejson': opts.dumpjson,
 770         'simulate': opts.simulate,
 771         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 772         'format': opts.format,
 773         'format_limit': opts.format_limit,
 774         'listformats': opts.listformats,
 775         'outtmpl': outtmpl,
 776         'autonumber_size': opts.autonumber_size,
 777         'restrictfilenames': opts.restrictfilenames,
 778         'ignoreerrors': opts.ignoreerrors,
 779         'ratelimit': opts.ratelimit,
 780         'sleepinterval': opts.sleepinterval,
 781         'nooverwrites': opts.nooverwrites,
 782         'retries': opts.retries,
 783         'buffersize': opts.buffersize,
 784         'noresizebuffer': opts.noresizebuffer,
 785         'continuedl': opts.continue_dl,
 786         'noprogress': opts.noprogress,
 787         'progress_with_newline': opts.progress_with_newline,
 788         'playliststart': opts.playliststart,
 789         'playlistend': opts.playlistend,
 790         'noplaylist': opts.noplaylist,
 791         'logtostderr': opts.outtmpl == '-',
 792         'consoletitle': opts.consoletitle,
 793         'nopart': opts.nopart,
 794         'updatetime': opts.updatetime,
 795         'writedescription': opts.writedescription,
 796         'writeannotations': opts.writeannotations,
 797         'writeinfojson': opts.writeinfojson,
 798         'writethumbnail': opts.writethumbnail,
 799         'writesubtitles': opts.writesubtitles,
 800         'writeautomaticsub': opts.writeautomaticsub,
 801         'allsubtitles': opts.allsubtitles,
 802         'listsubtitles': opts.listsubtitles,
 803         'subtitlesformat': opts.subtitlesformat,
 804         'subtitleslangs': opts.subtitleslangs,
 805         'matchtitle': decodeOption(opts.matchtitle),
 806         'rejecttitle': decodeOption(opts.rejecttitle),
 807         'max_downloads': opts.max_downloads,
 808         'prefer_free_formats': opts.prefer_free_formats,
 809         'verbose': opts.verbose,
 810         'dump_intermediate_pages': opts.dump_intermediate_pages,
 811         'write_pages': opts.write_pages,
 812         'test': opts.test,
 813         'keepvideo': opts.keepvideo,
 814         'min_filesize': opts.min_filesize,
 815         'max_filesize': opts.max_filesize,
 816         'min_views': opts.min_views,
 817         'max_views': opts.max_views,
 818         'daterange': date,
 819         'cachedir': opts.cachedir,
 820         'youtube_print_sig_code': opts.youtube_print_sig_code,
 821         'age_limit': opts.age_limit,
 822         'download_archive': download_archive_fn,
 823         'cookiefile': opts.cookiefile,
 824         'nocheckcertificate': opts.no_check_certificate,
 825         'prefer_insecure': opts.prefer_insecure,
 826         'proxy': opts.proxy,
 827         'socket_timeout': opts.socket_timeout,
 828         'bidi_workaround': opts.bidi_workaround,
 829         'debug_printtraffic': opts.debug_printtraffic,
 830         'prefer_ffmpeg': opts.prefer_ffmpeg,
 831         'include_ads': opts.include_ads,
 832         'default_search': opts.default_search,
 833         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 834         'encoding': opts.encoding,
 835     }
 836
 837     with YoutubeDL(ydl_opts) as ydl:
 838         ydl.print_debug_header()
 839         ydl.add_default_info_extractors()
 840
 841         # PostProcessors
 842         # Add the metadata pp first, the other pps will copy it
 843         if opts.addmetadata:
 844             ydl.add_post_processor(FFmpegMetadataPP())
 845         if opts.extractaudio:
 846             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 847         if opts.recodevideo:
 848             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 849         if opts.embedsubtitles:
 850             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 851         if opts.xattrs:
 852             ydl.add_post_processor(XAttrMetadataPP())
 853         if opts.embedthumbnail:
 854             if not opts.addmetadata:
 855                 ydl.add_post_processor(FFmpegAudioFixPP())
 856             ydl.add_post_processor(AtomicParsleyPP())
 857
 858         # Update version
 859         if opts.update_self:
 860             update_self(ydl.to_screen, opts.verbose)
 861
 862         # Remove cache dir
 863         if opts.rm_cachedir:
 864             if opts.cachedir is None:
 865                 ydl.to_screen(u'No cache dir specified (Did you combine --no-cache-dir and --rm-cache-dir?)')
 866             else:
 867                 if ('.cache' not in opts.cachedir) or ('youtube-dl' not in opts.cachedir):
 868                     ydl.to_screen(u'Not removing directory %s - this does not look like a cache dir')
 869                     retcode = 141
 870                 else:
 871                     ydl.to_screen(
 872                         u'Removing cache dir %s .' % opts.cachedir,
 873                         skip_eol=True)
 874                     if os.path.exists(opts.cachedir):
 875                         ydl.to_screen(u'.', skip_eol=True)
 876                         shutil.rmtree(opts.cachedir)
 877                     ydl.to_screen(u'.')
 878
 879         # Maybe do nothing
 880         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 881             if not (opts.update_self or opts.rm_cachedir):
 882                 parser.error(u'you must provide at least one URL')
 883             else:
 884                 sys.exit()
 885
 886         try:
 887             if opts.load_info_filename is not None:
 888                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 889             else:
 890                 retcode = ydl.download(all_urls)
 891         except MaxDownloadsReached:
 892             ydl.to_screen(u'--max-download limit reached, aborting.')
 893             retcode = 101
 894
 895     sys.exit(retcode)
 896
 897
 898 def main(argv=None):
 899     try:
 900         _real_main(argv)
 901     except DownloadError:
 902         sys.exit(1)
 903     except SameFileError:
 904         sys.exit(u'ERROR: fixed output name but more than one file to download')
 905     except KeyboardInterrupt:
 906         sys.exit(u'\nERROR: Interrupted by user')