youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53 )
  54
  55 __license__ = 'Public Domain'
  56
  57 import codecs
  58 import getpass
  59 import io
  60 import locale
  61 import optparse
  62 import os
  63 import random
  64 import re
  65 import shlex
  66 import sys
  67
  68
  69 from .utils import (
  70     compat_print,
  71     DateRange,
  72     decodeOption,
  73     get_term_width,
  74     DownloadError,
  75     get_cachedir,
  76     MaxDownloadsReached,
  77     preferredencoding,
  78     read_batch_urls,
  79     SameFileError,
  80     setproctitle,
  81     std_headers,
  82     write_string,
  83 )
  84 from .update import update_self
  85 from .FileDownloader import (
  86     FileDownloader,
  87 )
  88 from .extractor import gen_extractors
  89 from .version import __version__
  90 from .YoutubeDL import YoutubeDL
  91 from .postprocessor import (
  92     FFmpegMetadataPP,
  93     FFmpegVideoConvertor,
  94     FFmpegExtractAudioPP,
  95     FFmpegEmbedSubtitlePP,
  96     XAttrMetadataPP,
  97 )
  98
  99
 100 def parseOpts(overrideArguments=None):
 101     def _readOptions(filename_bytes, default=[]):
 102         try:
 103             optionf = open(filename_bytes)
 104         except IOError:
 105             return default  # silently skip if file is not present
 106         try:
 107             res = []
 108             for l in optionf:
 109                 res += shlex.split(l, comments=True)
 110         finally:
 111             optionf.close()
 112         return res
 113
 114     def _readUserConf():
 115         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 116         if xdg_config_home:
 117             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 118             if not os.path.isfile(userConfFile):
 119                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 120         else:
 121             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 122             if not os.path.isfile(userConfFile):
 123                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 124         userConf = _readOptions(userConfFile, None)
 125
 126         if userConf is None:
 127             appdata_dir = os.environ.get('appdata')
 128             if appdata_dir:
 129                 userConf = _readOptions(
 130                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 131                     default=None)
 132                 if userConf is None:
 133                     userConf = _readOptions(
 134                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 135                         default=None)
 136
 137         if userConf is None:
 138             userConf = _readOptions(
 139                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 140                 default=None)
 141         if userConf is None:
 142             userConf = _readOptions(
 143                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 144                 default=None)
 145
 146         if userConf is None:
 147             userConf = []
 148
 149         return userConf
 150
 151     def _format_option_string(option):
 152         ''' ('-o', '--option') -> -o, --format METAVAR'''
 153
 154         opts = []
 155
 156         if option._short_opts:
 157             opts.append(option._short_opts[0])
 158         if option._long_opts:
 159             opts.append(option._long_opts[0])
 160         if len(opts) > 1:
 161             opts.insert(1, ', ')
 162
 163         if option.takes_value(): opts.append(' %s' % option.metavar)
 164
 165         return "".join(opts)
 166
 167     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 168         setattr(parser.values, option.dest, value.split(','))
 169
 170     def _hide_login_info(opts):
 171         opts = list(opts)
 172         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 173             try:
 174                 i = opts.index(private_opt)
 175                 opts[i+1] = '<PRIVATE>'
 176             except ValueError:
 177                 pass
 178         return opts
 179
 180     max_width = 80
 181     max_help_position = 80
 182
 183     # No need to wrap help messages if we're on a wide console
 184     columns = get_term_width()
 185     if columns: max_width = columns
 186
 187     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 188     fmt.format_option_strings = _format_option_string
 189
 190     kw = {
 191         'version'   : __version__,
 192         'formatter' : fmt,
 193         'usage' : '%prog [options] url [url...]',
 194         'conflict_handler' : 'resolve',
 195     }
 196
 197     parser = optparse.OptionParser(**kw)
 198
 199     # option groups
 200     general        = optparse.OptionGroup(parser, 'General Options')
 201     selection      = optparse.OptionGroup(parser, 'Video Selection')
 202     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 203     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 204     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 205     downloader     = optparse.OptionGroup(parser, 'Download Options')
 206     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 207     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 208     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 209
 210     general.add_option('-h', '--help',
 211             action='help', help='print this help text and exit')
 212     general.add_option('-v', '--version',
 213             action='version', help='print program version and exit')
 214     general.add_option('-U', '--update',
 215             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 216     general.add_option('-i', '--ignore-errors',
 217             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 218     general.add_option('--abort-on-error',
 219             action='store_false', dest='ignoreerrors',
 220             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 221     general.add_option('--dump-user-agent',
 222             action='store_true', dest='dump_user_agent',
 223             help='display the current browser identification', default=False)
 224     general.add_option('--user-agent',
 225             dest='user_agent', help='specify a custom user agent', metavar='UA')
 226     general.add_option('--referer',
 227             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 228             metavar='REF', default=None)
 229     general.add_option('--list-extractors',
 230             action='store_true', dest='list_extractors',
 231             help='List all supported extractors and the URLs they would handle', default=False)
 232     general.add_option('--extractor-descriptions',
 233             action='store_true', dest='list_extractor_descriptions',
 234             help='Output descriptions of all supported extractors', default=False)
 235     general.add_option(
 236         '--proxy', dest='proxy', default=None, metavar='URL',
 237         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 238     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 239     general.add_option(
 240         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 241         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 242     general.add_option(
 243         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 244         help='Disable filesystem caching')
 245     general.add_option(
 246         '--socket-timeout', dest='socket_timeout',
 247         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 248     general.add_option(
 249         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 250         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 251     general.add_option('--default-search',
 252             dest='default_search', metavar='PREFIX',
 253             help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". By default (with value "auto") youtube-dl guesses.')
 254     general.add_option(
 255         '--ignore-config',
 256         action='store_true',
 257         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 258
 259
 260     selection.add_option(
 261         '--playlist-start',
 262         dest='playliststart', metavar='NUMBER', default=1, type=int,
 263         help='playlist video to start at (default is %default)')
 264     selection.add_option(
 265         '--playlist-end',
 266         dest='playlistend', metavar='NUMBER', default=None, type=int,
 267         help='playlist video to end at (default is last)')
 268     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 269     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 270     selection.add_option('--max-downloads', metavar='NUMBER',
 271                          dest='max_downloads', type=int, default=None,
 272                          help='Abort after downloading NUMBER files')
 273     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 274     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 275     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 276     selection.add_option(
 277         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 278         help='download only videos uploaded on or before this date (i.e. inclusive)')
 279     selection.add_option(
 280         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 281         help='download only videos uploaded on or after this date (i.e. inclusive)')
 282     selection.add_option(
 283         '--min-views', metavar='COUNT', dest='min_views',
 284         default=None, type=int,
 285         help="Do not download any videos with less than COUNT views",)
 286     selection.add_option(
 287         '--max-views', metavar='COUNT', dest='max_views',
 288         default=None, type=int,
 289         help="Do not download any videos with more than COUNT views",)
 290     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 291     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 292                          help='download only videos suitable for the given age',
 293                          default=None, type=int)
 294     selection.add_option('--download-archive', metavar='FILE',
 295                          dest='download_archive',
 296                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 297     selection.add_option(
 298         '--include-ads', dest='include_ads',
 299         action='store_true',
 300         help='Download advertisements as well (experimental)')
 301     selection.add_option(
 302         '--youtube-include-dash-manifest', action='store_true',
 303         dest='youtube_include_dash_manifest', default=False,
 304         help='Try to download the DASH manifest on YouTube videos (experimental)')
 305
 306     authentication.add_option('-u', '--username',
 307             dest='username', metavar='USERNAME', help='account username')
 308     authentication.add_option('-p', '--password',
 309             dest='password', metavar='PASSWORD', help='account password')
 310     authentication.add_option('-n', '--netrc',
 311             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 312     authentication.add_option('--video-password',
 313             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 314
 315
 316     video_format.add_option('-f', '--format',
 317             action='store', dest='format', metavar='FORMAT', default=None,
 318             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestaudio", "worst", and "worstaudio". By default, youtube-dl will pick the best quality.')
 319     video_format.add_option('--all-formats',
 320             action='store_const', dest='format', help='download all available video formats', const='all')
 321     video_format.add_option('--prefer-free-formats',
 322             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 323     video_format.add_option('--max-quality',
 324             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 325     video_format.add_option('-F', '--list-formats',
 326             action='store_true', dest='listformats', help='list all available formats')
 327
 328     subtitles.add_option('--write-sub', '--write-srt',
 329             action='store_true', dest='writesubtitles',
 330             help='write subtitle file', default=False)
 331     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 332             action='store_true', dest='writeautomaticsub',
 333             help='write automatic subtitle file (youtube only)', default=False)
 334     subtitles.add_option('--all-subs',
 335             action='store_true', dest='allsubtitles',
 336             help='downloads all the available subtitles of the video', default=False)
 337     subtitles.add_option('--list-subs',
 338             action='store_true', dest='listsubtitles',
 339             help='lists all available subtitles for the video', default=False)
 340     subtitles.add_option('--sub-format',
 341             action='store', dest='subtitlesformat', metavar='FORMAT',
 342             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 343     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 344             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 345             default=[], callback=_comma_separated_values_options_callback,
 346             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 347
 348     downloader.add_option('-r', '--rate-limit',
 349             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 350     downloader.add_option('-R', '--retries',
 351             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 352     downloader.add_option('--buffer-size',
 353             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 354     downloader.add_option('--no-resize-buffer',
 355             action='store_true', dest='noresizebuffer',
 356             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 357     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 358
 359     verbosity.add_option('-q', '--quiet',
 360             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 361     verbosity.add_option('-s', '--simulate',
 362             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 363     verbosity.add_option('--skip-download',
 364             action='store_true', dest='skip_download', help='do not download the video', default=False)
 365     verbosity.add_option('-g', '--get-url',
 366             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 367     verbosity.add_option('-e', '--get-title',
 368             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 369     verbosity.add_option('--get-id',
 370             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 371     verbosity.add_option('--get-thumbnail',
 372             action='store_true', dest='getthumbnail',
 373             help='simulate, quiet but print thumbnail URL', default=False)
 374     verbosity.add_option('--get-description',
 375             action='store_true', dest='getdescription',
 376             help='simulate, quiet but print video description', default=False)
 377     verbosity.add_option('--get-duration',
 378             action='store_true', dest='getduration',
 379             help='simulate, quiet but print video length', default=False)
 380     verbosity.add_option('--get-filename',
 381             action='store_true', dest='getfilename',
 382             help='simulate, quiet but print output filename', default=False)
 383     verbosity.add_option('--get-format',
 384             action='store_true', dest='getformat',
 385             help='simulate, quiet but print output format', default=False)
 386     verbosity.add_option('-j', '--dump-json',
 387             action='store_true', dest='dumpjson',
 388             help='simulate, quiet but print JSON information', default=False)
 389     verbosity.add_option('--newline',
 390             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 391     verbosity.add_option('--no-progress',
 392             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 393     verbosity.add_option('--console-title',
 394             action='store_true', dest='consoletitle',
 395             help='display progress in console titlebar', default=False)
 396     verbosity.add_option('-v', '--verbose',
 397             action='store_true', dest='verbose', help='print various debugging information', default=False)
 398     verbosity.add_option('--dump-intermediate-pages',
 399             action='store_true', dest='dump_intermediate_pages', default=False,
 400             help='print downloaded pages to debug problems (very verbose)')
 401     verbosity.add_option('--write-pages',
 402             action='store_true', dest='write_pages', default=False,
 403             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 404     verbosity.add_option('--youtube-print-sig-code',
 405             action='store_true', dest='youtube_print_sig_code', default=False,
 406             help=optparse.SUPPRESS_HELP)
 407     verbosity.add_option('--print-traffic',
 408             dest='debug_printtraffic', action='store_true', default=False,
 409             help='Display sent and read HTTP traffic')
 410
 411
 412     filesystem.add_option('-t', '--title',
 413             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 414     filesystem.add_option('--id',
 415             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 416     filesystem.add_option('-l', '--literal',
 417             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 418     filesystem.add_option('-A', '--auto-number',
 419             action='store_true', dest='autonumber',
 420             help='number downloaded files starting from 00000', default=False)
 421     filesystem.add_option('-o', '--output',
 422             dest='outtmpl', metavar='TEMPLATE',
 423             help=('output filename template. Use %(title)s to get the title, '
 424                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 425                   '%(autonumber)s to get an automatically incremented number, '
 426                   '%(ext)s for the filename extension, '
 427                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 428                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 429                   '%(upload_date)s for the upload date (YYYYMMDD), '
 430                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 431                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 432                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 433                   '%(height)s and %(width)s for the width and height of the video format. '
 434                   '%(resolution)s for a textual description of the resolution of the video format. '
 435                   'Use - to output to stdout. Can also be used to download to a different directory, '
 436                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 437     filesystem.add_option('--autonumber-size',
 438             dest='autonumber_size', metavar='NUMBER',
 439             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 440     filesystem.add_option('--restrict-filenames',
 441             action='store_true', dest='restrictfilenames',
 442             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 443     filesystem.add_option('-a', '--batch-file',
 444             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 445     filesystem.add_option('--load-info',
 446             dest='load_info_filename', metavar='FILE',
 447             help='json file containing the video information (created with the "--write-json" option)')
 448     filesystem.add_option('-w', '--no-overwrites',
 449             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 450     filesystem.add_option('-c', '--continue',
 451             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 452     filesystem.add_option('--no-continue',
 453             action='store_false', dest='continue_dl',
 454             help='do not resume partially downloaded files (restart from beginning)')
 455     filesystem.add_option('--cookies',
 456             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 457     filesystem.add_option('--no-part',
 458             action='store_true', dest='nopart', help='do not use .part files', default=False)
 459     filesystem.add_option('--no-mtime',
 460             action='store_false', dest='updatetime',
 461             help='do not use the Last-modified header to set the file modification time', default=True)
 462     filesystem.add_option('--write-description',
 463             action='store_true', dest='writedescription',
 464             help='write video description to a .description file', default=False)
 465     filesystem.add_option('--write-info-json',
 466             action='store_true', dest='writeinfojson',
 467             help='write video metadata to a .info.json file', default=False)
 468     filesystem.add_option('--write-annotations',
 469             action='store_true', dest='writeannotations',
 470             help='write video annotations to a .annotation file', default=False)
 471     filesystem.add_option('--write-thumbnail',
 472             action='store_true', dest='writethumbnail',
 473             help='write thumbnail image to disk', default=False)
 474
 475
 476     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 477             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 478     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 479             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 480     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 481             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 482     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 483             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 484     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 485             help='keeps the video file on disk after the post-processing; the video is erased by default')
 486     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 487             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 488     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 489             help='embed subtitles in the video (only for mp4 videos)')
 490     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 491             help='write metadata to the video file')
 492     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 493             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 494     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 495         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 496     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 497         help='Prefer ffmpeg over avconv for running the postprocessors')
 498
 499
 500     parser.add_option_group(general)
 501     parser.add_option_group(selection)
 502     parser.add_option_group(downloader)
 503     parser.add_option_group(filesystem)
 504     parser.add_option_group(verbosity)
 505     parser.add_option_group(video_format)
 506     parser.add_option_group(subtitles)
 507     parser.add_option_group(authentication)
 508     parser.add_option_group(postproc)
 509
 510     if overrideArguments is not None:
 511         opts, args = parser.parse_args(overrideArguments)
 512         if opts.verbose:
 513             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 514     else:
 515         commandLineConf = sys.argv[1:]
 516         if '--ignore-config' in commandLineConf:
 517             systemConf = []
 518             userConf = []
 519         else:
 520             systemConf = _readOptions('/etc/youtube-dl.conf')
 521             if '--ignore-config' in systemConf:
 522                 userConf = []
 523             else:
 524                 userConf = _readUserConf()
 525         argv = systemConf + userConf + commandLineConf
 526
 527         opts, args = parser.parse_args(argv)
 528         if opts.verbose:
 529             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 530             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 531             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 532             write_string(u'[debug] Encodings: locale %r, fs %r, out %r, pref: %r\n' %
 533                          (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, preferredencoding()))
 534
 535     return parser, opts, args
 536
 537
 538 def _real_main(argv=None):
 539     # Compatibility fixes for Windows
 540     if sys.platform == 'win32':
 541         # https://github.com/rg3/youtube-dl/issues/820
 542         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 543
 544     setproctitle(u'youtube-dl')
 545
 546     parser, opts, args = parseOpts(argv)
 547
 548     # Set user agent
 549     if opts.user_agent is not None:
 550         std_headers['User-Agent'] = opts.user_agent
 551
 552     # Set referer
 553     if opts.referer is not None:
 554         std_headers['Referer'] = opts.referer
 555
 556     # Dump user agent
 557     if opts.dump_user_agent:
 558         compat_print(std_headers['User-Agent'])
 559         sys.exit(0)
 560
 561     # Batch file verification
 562     batch_urls = []
 563     if opts.batchfile is not None:
 564         try:
 565             if opts.batchfile == '-':
 566                 batchfd = sys.stdin
 567             else:
 568                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 569             batch_urls = read_batch_urls(batchfd)
 570             if opts.verbose:
 571                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 572         except IOError:
 573             sys.exit(u'ERROR: batch file could not be read')
 574     all_urls = batch_urls + args
 575     all_urls = [url.strip() for url in all_urls]
 576     _enc = preferredencoding()
 577     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 578
 579     extractors = gen_extractors()
 580
 581     if opts.list_extractors:
 582         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 583             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 584             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 585             for mu in matchedUrls:
 586                 compat_print(u'  ' + mu)
 587         sys.exit(0)
 588     if opts.list_extractor_descriptions:
 589         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 590             if not ie._WORKING:
 591                 continue
 592             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 593             if desc is False:
 594                 continue
 595             if hasattr(ie, 'SEARCH_KEY'):
 596                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 597                 _COUNTS = (u'', u'5', u'10', u'all')
 598                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 599             compat_print(desc)
 600         sys.exit(0)
 601
 602
 603     # Conflicting, missing and erroneous options
 604     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 605         parser.error(u'using .netrc conflicts with giving username/password')
 606     if opts.password is not None and opts.username is None:
 607         parser.error(u'account username missing\n')
 608     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 609         parser.error(u'using output template conflicts with using title, video ID or auto number')
 610     if opts.usetitle and opts.useid:
 611         parser.error(u'using title conflicts with using video ID')
 612     if opts.username is not None and opts.password is None:
 613         opts.password = getpass.getpass(u'Type account password and press return:')
 614     if opts.ratelimit is not None:
 615         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 616         if numeric_limit is None:
 617             parser.error(u'invalid rate limit specified')
 618         opts.ratelimit = numeric_limit
 619     if opts.min_filesize is not None:
 620         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 621         if numeric_limit is None:
 622             parser.error(u'invalid min_filesize specified')
 623         opts.min_filesize = numeric_limit
 624     if opts.max_filesize is not None:
 625         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 626         if numeric_limit is None:
 627             parser.error(u'invalid max_filesize specified')
 628         opts.max_filesize = numeric_limit
 629     if opts.retries is not None:
 630         try:
 631             opts.retries = int(opts.retries)
 632         except (TypeError, ValueError):
 633             parser.error(u'invalid retry count specified')
 634     if opts.buffersize is not None:
 635         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 636         if numeric_buffersize is None:
 637             parser.error(u'invalid buffer size specified')
 638         opts.buffersize = numeric_buffersize
 639     if opts.playliststart <= 0:
 640         raise ValueError(u'Playlist start must be positive')
 641     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 642         raise ValueError(u'Playlist end must be greater than playlist start')
 643     if opts.extractaudio:
 644         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 645             parser.error(u'invalid audio format specified')
 646     if opts.audioquality:
 647         opts.audioquality = opts.audioquality.strip('k').strip('K')
 648         if not opts.audioquality.isdigit():
 649             parser.error(u'invalid audio quality specified')
 650     if opts.recodevideo is not None:
 651         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 652             parser.error(u'invalid video recode format specified')
 653     if opts.date is not None:
 654         date = DateRange.day(opts.date)
 655     else:
 656         date = DateRange(opts.dateafter, opts.datebefore)
 657     if opts.default_search not in ('auto', None) and ':' not in opts.default_search:
 658         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 659
 660     # Do not download videos when there are audio-only formats
 661     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 662         opts.format = 'bestaudio/best'
 663
 664     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 665     # this was the old behaviour if only --all-sub was given.
 666     if opts.allsubtitles and (opts.writeautomaticsub == False):
 667         opts.writesubtitles = True
 668
 669     if sys.version_info < (3,):
 670         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 671         if opts.outtmpl is not None:
 672             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 673     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 674             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 675             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 676             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 677             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 678             or (opts.useid and u'%(id)s.%(ext)s')
 679             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 680             or u'%(title)s-%(id)s.%(ext)s')
 681     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 682         parser.error(u'Cannot download a video and extract audio into the same'
 683                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 684                      u' template'.format(outtmpl))
 685
 686     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 687     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 688
 689     ydl_opts = {
 690         'usenetrc': opts.usenetrc,
 691         'username': opts.username,
 692         'password': opts.password,
 693         'videopassword': opts.videopassword,
 694         'quiet': (opts.quiet or any_printing),
 695         'forceurl': opts.geturl,
 696         'forcetitle': opts.gettitle,
 697         'forceid': opts.getid,
 698         'forcethumbnail': opts.getthumbnail,
 699         'forcedescription': opts.getdescription,
 700         'forceduration': opts.getduration,
 701         'forcefilename': opts.getfilename,
 702         'forceformat': opts.getformat,
 703         'forcejson': opts.dumpjson,
 704         'simulate': opts.simulate,
 705         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 706         'format': opts.format,
 707         'format_limit': opts.format_limit,
 708         'listformats': opts.listformats,
 709         'outtmpl': outtmpl,
 710         'autonumber_size': opts.autonumber_size,
 711         'restrictfilenames': opts.restrictfilenames,
 712         'ignoreerrors': opts.ignoreerrors,
 713         'ratelimit': opts.ratelimit,
 714         'nooverwrites': opts.nooverwrites,
 715         'retries': opts.retries,
 716         'buffersize': opts.buffersize,
 717         'noresizebuffer': opts.noresizebuffer,
 718         'continuedl': opts.continue_dl,
 719         'noprogress': opts.noprogress,
 720         'progress_with_newline': opts.progress_with_newline,
 721         'playliststart': opts.playliststart,
 722         'playlistend': opts.playlistend,
 723         'noplaylist': opts.noplaylist,
 724         'logtostderr': opts.outtmpl == '-',
 725         'consoletitle': opts.consoletitle,
 726         'nopart': opts.nopart,
 727         'updatetime': opts.updatetime,
 728         'writedescription': opts.writedescription,
 729         'writeannotations': opts.writeannotations,
 730         'writeinfojson': opts.writeinfojson,
 731         'writethumbnail': opts.writethumbnail,
 732         'writesubtitles': opts.writesubtitles,
 733         'writeautomaticsub': opts.writeautomaticsub,
 734         'allsubtitles': opts.allsubtitles,
 735         'listsubtitles': opts.listsubtitles,
 736         'subtitlesformat': opts.subtitlesformat,
 737         'subtitleslangs': opts.subtitleslangs,
 738         'matchtitle': decodeOption(opts.matchtitle),
 739         'rejecttitle': decodeOption(opts.rejecttitle),
 740         'max_downloads': opts.max_downloads,
 741         'prefer_free_formats': opts.prefer_free_formats,
 742         'verbose': opts.verbose,
 743         'dump_intermediate_pages': opts.dump_intermediate_pages,
 744         'write_pages': opts.write_pages,
 745         'test': opts.test,
 746         'keepvideo': opts.keepvideo,
 747         'min_filesize': opts.min_filesize,
 748         'max_filesize': opts.max_filesize,
 749         'min_views': opts.min_views,
 750         'max_views': opts.max_views,
 751         'daterange': date,
 752         'cachedir': opts.cachedir,
 753         'youtube_print_sig_code': opts.youtube_print_sig_code,
 754         'age_limit': opts.age_limit,
 755         'download_archive': download_archive_fn,
 756         'cookiefile': opts.cookiefile,
 757         'nocheckcertificate': opts.no_check_certificate,
 758         'proxy': opts.proxy,
 759         'socket_timeout': opts.socket_timeout,
 760         'bidi_workaround': opts.bidi_workaround,
 761         'debug_printtraffic': opts.debug_printtraffic,
 762         'prefer_ffmpeg': opts.prefer_ffmpeg,
 763         'include_ads': opts.include_ads,
 764         'default_search': opts.default_search,
 765         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 766     }
 767
 768     with YoutubeDL(ydl_opts) as ydl:
 769         ydl.print_debug_header()
 770         ydl.add_default_info_extractors()
 771
 772         # PostProcessors
 773         # Add the metadata pp first, the other pps will copy it
 774         if opts.addmetadata:
 775             ydl.add_post_processor(FFmpegMetadataPP())
 776         if opts.extractaudio:
 777             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 778         if opts.recodevideo:
 779             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 780         if opts.embedsubtitles:
 781             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 782         if opts.xattrs:
 783             ydl.add_post_processor(XAttrMetadataPP())
 784
 785         # Update version
 786         if opts.update_self:
 787             update_self(ydl.to_screen, opts.verbose)
 788
 789         # Maybe do nothing
 790         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 791             if not opts.update_self:
 792                 parser.error(u'you must provide at least one URL')
 793             else:
 794                 sys.exit()
 795
 796         try:
 797             if opts.load_info_filename is not None:
 798                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 799             else:
 800                 retcode = ydl.download(all_urls)
 801         except MaxDownloadsReached:
 802             ydl.to_screen(u'--max-download limit reached, aborting.')
 803             retcode = 101
 804
 805     sys.exit(retcode)
 806
 807
 808 def main(argv=None):
 809     try:
 810         _real_main(argv)
 811     except DownloadError:
 812         sys.exit(1)
 813     except SameFileError:
 814         sys.exit(u'ERROR: fixed output name but more than one file to download')
 815     except KeyboardInterrupt:
 816         sys.exit(u'\nERROR: Interrupted by user')