youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43 )
  44
  45 __license__ = 'Public Domain'
  46
  47 import codecs
  48 import getpass
  49 import locale
  50 import optparse
  51 import os
  52 import random
  53 import re
  54 import shlex
  55 import sys
  56
  57
  58 from .utils import (
  59     compat_print,
  60     DateRange,
  61     decodeOption,
  62     get_term_width,
  63     DownloadError,
  64     get_cachedir,
  65     MaxDownloadsReached,
  66     preferredencoding,
  67     SameFileError,
  68     setproctitle,
  69     std_headers,
  70     write_string,
  71 )
  72 from .update import update_self
  73 from .FileDownloader import (
  74     FileDownloader,
  75 )
  76 from .extractor import gen_extractors
  77 from .version import __version__
  78 from .YoutubeDL import YoutubeDL
  79 from .postprocessor import (
  80     FFmpegMetadataPP,
  81     FFmpegVideoConvertor,
  82     FFmpegExtractAudioPP,
  83     FFmpegEmbedSubtitlePP,
  84     XAttrMetadataPP,
  85 )
  86
  87
  88 def parseOpts(overrideArguments=None):
  89     def _readOptions(filename_bytes, default=[]):
  90         try:
  91             optionf = open(filename_bytes)
  92         except IOError:
  93             return default  # silently skip if file is not present
  94         try:
  95             res = []
  96             for l in optionf:
  97                 res += shlex.split(l, comments=True)
  98         finally:
  99             optionf.close()
 100         return res
 101
 102     def _format_option_string(option):
 103         ''' ('-o', '--option') -> -o, --format METAVAR'''
 104
 105         opts = []
 106
 107         if option._short_opts:
 108             opts.append(option._short_opts[0])
 109         if option._long_opts:
 110             opts.append(option._long_opts[0])
 111         if len(opts) > 1:
 112             opts.insert(1, ', ')
 113
 114         if option.takes_value(): opts.append(' %s' % option.metavar)
 115
 116         return "".join(opts)
 117
 118     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 119         setattr(parser.values, option.dest, value.split(','))
 120
 121     def _hide_login_info(opts):
 122         opts = list(opts)
 123         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 124             try:
 125                 i = opts.index(private_opt)
 126                 opts[i+1] = '<PRIVATE>'
 127             except ValueError:
 128                 pass
 129         return opts
 130
 131     max_width = 80
 132     max_help_position = 80
 133
 134     # No need to wrap help messages if we're on a wide console
 135     columns = get_term_width()
 136     if columns: max_width = columns
 137
 138     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 139     fmt.format_option_strings = _format_option_string
 140
 141     kw = {
 142         'version'   : __version__,
 143         'formatter' : fmt,
 144         'usage' : '%prog [options] url [url...]',
 145         'conflict_handler' : 'resolve',
 146     }
 147
 148     parser = optparse.OptionParser(**kw)
 149
 150     # option groups
 151     general        = optparse.OptionGroup(parser, 'General Options')
 152     selection      = optparse.OptionGroup(parser, 'Video Selection')
 153     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 154     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 155     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 156     downloader     = optparse.OptionGroup(parser, 'Download Options')
 157     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 158     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 159     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 160
 161     general.add_option('-h', '--help',
 162             action='help', help='print this help text and exit')
 163     general.add_option('-v', '--version',
 164             action='version', help='print program version and exit')
 165     general.add_option('-U', '--update',
 166             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 167     general.add_option('-i', '--ignore-errors',
 168             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 169     general.add_option('--abort-on-error',
 170             action='store_false', dest='ignoreerrors',
 171             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 172     general.add_option('--dump-user-agent',
 173             action='store_true', dest='dump_user_agent',
 174             help='display the current browser identification', default=False)
 175     general.add_option('--user-agent',
 176             dest='user_agent', help='specify a custom user agent', metavar='UA')
 177     general.add_option('--referer',
 178             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 179             metavar='REF', default=None)
 180     general.add_option('--list-extractors',
 181             action='store_true', dest='list_extractors',
 182             help='List all supported extractors and the URLs they would handle', default=False)
 183     general.add_option('--extractor-descriptions',
 184             action='store_true', dest='list_extractor_descriptions',
 185             help='Output descriptions of all supported extractors', default=False)
 186     general.add_option(
 187         '--proxy', dest='proxy', default=None, metavar='URL',
 188         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 189     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 190     general.add_option(
 191         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 192         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 193     general.add_option(
 194         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 195         help='Disable filesystem caching')
 196     general.add_option(
 197         '--socket-timeout', dest='socket_timeout',
 198         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 199     general.add_option(
 200         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 201         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 202
 203
 204     selection.add_option(
 205         '--playlist-start',
 206         dest='playliststart', metavar='NUMBER', default=1, type=int,
 207         help='playlist video to start at (default is %default)')
 208     selection.add_option(
 209         '--playlist-end',
 210         dest='playlistend', metavar='NUMBER', default=None, type=int,
 211         help='playlist video to end at (default is last)')
 212     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 213     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 214     selection.add_option('--max-downloads', metavar='NUMBER',
 215                          dest='max_downloads', type=int, default=None,
 216                          help='Abort after downloading NUMBER files')
 217     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 218     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 219     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 220     selection.add_option(
 221         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 222         help='download only videos uploaded on or before this date (i.e. inclusive)')
 223     selection.add_option(
 224         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 225         help='download only videos uploaded on or after this date (i.e. inclusive)')
 226     selection.add_option(
 227         '--min-views', metavar='COUNT', dest='min_views',
 228         default=None, type=int,
 229         help="Do not download any videos with less than COUNT views",)
 230     selection.add_option(
 231         '--max-views', metavar='COUNT', dest='max_views',
 232         default=None, type=int,
 233         help="Do not download any videos with more than COUNT views",)
 234     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 235     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 236                          help='download only videos suitable for the given age',
 237                          default=None, type=int)
 238     selection.add_option('--download-archive', metavar='FILE',
 239                          dest='download_archive',
 240                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 241     selection.add_option(
 242         '--include-ads', dest='include_ads',
 243         action='store_true',
 244         help='Download advertisements as well (experimental)')
 245
 246     authentication.add_option('-u', '--username',
 247             dest='username', metavar='USERNAME', help='account username')
 248     authentication.add_option('-p', '--password',
 249             dest='password', metavar='PASSWORD', help='account password')
 250     authentication.add_option('-n', '--netrc',
 251             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 252     authentication.add_option('--video-password',
 253             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 254
 255
 256     video_format.add_option('-f', '--format',
 257             action='store', dest='format', metavar='FORMAT', default='best',
 258             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
 259     video_format.add_option('--all-formats',
 260             action='store_const', dest='format', help='download all available video formats', const='all')
 261     video_format.add_option('--prefer-free-formats',
 262             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 263     video_format.add_option('--max-quality',
 264             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 265     video_format.add_option('-F', '--list-formats',
 266             action='store_true', dest='listformats', help='list all available formats')
 267
 268     subtitles.add_option('--write-sub', '--write-srt',
 269             action='store_true', dest='writesubtitles',
 270             help='write subtitle file', default=False)
 271     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 272             action='store_true', dest='writeautomaticsub',
 273             help='write automatic subtitle file (youtube only)', default=False)
 274     subtitles.add_option('--all-subs',
 275             action='store_true', dest='allsubtitles',
 276             help='downloads all the available subtitles of the video', default=False)
 277     subtitles.add_option('--list-subs',
 278             action='store_true', dest='listsubtitles',
 279             help='lists all available subtitles for the video', default=False)
 280     subtitles.add_option('--sub-format',
 281             action='store', dest='subtitlesformat', metavar='FORMAT',
 282             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 283     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 284             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 285             default=[], callback=_comma_separated_values_options_callback,
 286             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 287
 288     downloader.add_option('-r', '--rate-limit',
 289             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 290     downloader.add_option('-R', '--retries',
 291             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 292     downloader.add_option('--buffer-size',
 293             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 294     downloader.add_option('--no-resize-buffer',
 295             action='store_true', dest='noresizebuffer',
 296             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 297     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 298
 299     verbosity.add_option('-q', '--quiet',
 300             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 301     verbosity.add_option('-s', '--simulate',
 302             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 303     verbosity.add_option('--skip-download',
 304             action='store_true', dest='skip_download', help='do not download the video', default=False)
 305     verbosity.add_option('-g', '--get-url',
 306             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 307     verbosity.add_option('-e', '--get-title',
 308             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 309     verbosity.add_option('--get-id',
 310             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 311     verbosity.add_option('--get-thumbnail',
 312             action='store_true', dest='getthumbnail',
 313             help='simulate, quiet but print thumbnail URL', default=False)
 314     verbosity.add_option('--get-description',
 315             action='store_true', dest='getdescription',
 316             help='simulate, quiet but print video description', default=False)
 317     verbosity.add_option('--get-duration',
 318             action='store_true', dest='getduration',
 319             help='simulate, quiet but print video length', default=False)
 320     verbosity.add_option('--get-filename',
 321             action='store_true', dest='getfilename',
 322             help='simulate, quiet but print output filename', default=False)
 323     verbosity.add_option('--get-format',
 324             action='store_true', dest='getformat',
 325             help='simulate, quiet but print output format', default=False)
 326     verbosity.add_option('-j', '--dump-json',
 327             action='store_true', dest='dumpjson',
 328             help='simulate, quiet but print JSON information', default=False)
 329     verbosity.add_option('--newline',
 330             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 331     verbosity.add_option('--no-progress',
 332             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 333     verbosity.add_option('--console-title',
 334             action='store_true', dest='consoletitle',
 335             help='display progress in console titlebar', default=False)
 336     verbosity.add_option('-v', '--verbose',
 337             action='store_true', dest='verbose', help='print various debugging information', default=False)
 338     verbosity.add_option('--dump-intermediate-pages',
 339             action='store_true', dest='dump_intermediate_pages', default=False,
 340             help='print downloaded pages to debug problems (very verbose)')
 341     verbosity.add_option('--write-pages',
 342             action='store_true', dest='write_pages', default=False,
 343             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 344     verbosity.add_option('--youtube-print-sig-code',
 345             action='store_true', dest='youtube_print_sig_code', default=False,
 346             help=optparse.SUPPRESS_HELP)
 347     verbosity.add_option('--print-traffic',
 348             dest='debug_printtraffic', action='store_true', default=False,
 349             help=optparse.SUPPRESS_HELP)
 350
 351     filesystem.add_option('-t', '--title',
 352             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 353     filesystem.add_option('--id',
 354             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 355     filesystem.add_option('-l', '--literal',
 356             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 357     filesystem.add_option('-A', '--auto-number',
 358             action='store_true', dest='autonumber',
 359             help='number downloaded files starting from 00000', default=False)
 360     filesystem.add_option('-o', '--output',
 361             dest='outtmpl', metavar='TEMPLATE',
 362             help=('output filename template. Use %(title)s to get the title, '
 363                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 364                   '%(autonumber)s to get an automatically incremented number, '
 365                   '%(ext)s for the filename extension, '
 366                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 367                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 368                   '%(upload_date)s for the upload date (YYYYMMDD), '
 369                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 370                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 371                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 372                   'Use - to output to stdout. Can also be used to download to a different directory, '
 373                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 374     filesystem.add_option('--autonumber-size',
 375             dest='autonumber_size', metavar='NUMBER',
 376             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 377     filesystem.add_option('--restrict-filenames',
 378             action='store_true', dest='restrictfilenames',
 379             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 380     filesystem.add_option('-a', '--batch-file',
 381             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 382     filesystem.add_option('--load-info',
 383             dest='load_info_filename', metavar='FILE',
 384             help='json file containing the video information (created with the "--write-json" option)')
 385     filesystem.add_option('-w', '--no-overwrites',
 386             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 387     filesystem.add_option('-c', '--continue',
 388             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 389     filesystem.add_option('--no-continue',
 390             action='store_false', dest='continue_dl',
 391             help='do not resume partially downloaded files (restart from beginning)')
 392     filesystem.add_option('--cookies',
 393             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 394     filesystem.add_option('--no-part',
 395             action='store_true', dest='nopart', help='do not use .part files', default=False)
 396     filesystem.add_option('--no-mtime',
 397             action='store_false', dest='updatetime',
 398             help='do not use the Last-modified header to set the file modification time', default=True)
 399     filesystem.add_option('--write-description',
 400             action='store_true', dest='writedescription',
 401             help='write video description to a .description file', default=False)
 402     filesystem.add_option('--write-info-json',
 403             action='store_true', dest='writeinfojson',
 404             help='write video metadata to a .info.json file', default=False)
 405     filesystem.add_option('--write-annotations',
 406             action='store_true', dest='writeannotations',
 407             help='write video annotations to a .annotation file', default=False)
 408     filesystem.add_option('--write-thumbnail',
 409             action='store_true', dest='writethumbnail',
 410             help='write thumbnail image to disk', default=False)
 411
 412
 413     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 414             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 415     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 416             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 417     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 418             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 419     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 420             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 421     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 422             help='keeps the video file on disk after the post-processing; the video is erased by default')
 423     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 424             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 425     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 426             help='embed subtitles in the video (only for mp4 videos)')
 427     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 428             help='write metadata to the video file')
 429     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 430             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 431     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 432         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 433     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 434         help='Prefer ffmpeg over avconv for running the postprocessors')
 435
 436
 437     parser.add_option_group(general)
 438     parser.add_option_group(selection)
 439     parser.add_option_group(downloader)
 440     parser.add_option_group(filesystem)
 441     parser.add_option_group(verbosity)
 442     parser.add_option_group(video_format)
 443     parser.add_option_group(subtitles)
 444     parser.add_option_group(authentication)
 445     parser.add_option_group(postproc)
 446
 447     if overrideArguments is not None:
 448         opts, args = parser.parse_args(overrideArguments)
 449         if opts.verbose:
 450             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 451     else:
 452         systemConf = _readOptions('/etc/youtube-dl.conf')
 453
 454         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 455         if xdg_config_home:
 456             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 457             if not os.path.isfile(userConfFile):
 458                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 459         else:
 460             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 461             if not os.path.isfile(userConfFile):
 462                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 463         userConf = _readOptions(userConfFile, None)
 464
 465         if userConf is None:
 466             appdata_dir = os.environ.get('appdata')
 467             if appdata_dir:
 468                 userConf = _readOptions(
 469                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 470                     default=None)
 471                 if userConf is None:
 472                     userConf = _readOptions(
 473                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 474                         default=None)
 475
 476         if userConf is None:
 477             userConf = _readOptions(
 478                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 479                 default=None)
 480         if userConf is None:
 481             userConf = _readOptions(
 482                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 483                 default=None)
 484
 485         if userConf is None:
 486             userConf = []
 487
 488         commandLineConf = sys.argv[1:]
 489         argv = systemConf + userConf + commandLineConf
 490         opts, args = parser.parse_args(argv)
 491         if opts.verbose:
 492             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 493             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 494             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 495             write_string(u'[debug] Encodings: locale %r, fs %r, out %r, pref: %r\n' %
 496                          (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, preferredencoding()))
 497
 498     return parser, opts, args
 499
 500
 501 def _real_main(argv=None):
 502     # Compatibility fixes for Windows
 503     if sys.platform == 'win32':
 504         # https://github.com/rg3/youtube-dl/issues/820
 505         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 506
 507     setproctitle(u'youtube-dl')
 508
 509     parser, opts, args = parseOpts(argv)
 510
 511     # Set user agent
 512     if opts.user_agent is not None:
 513         std_headers['User-Agent'] = opts.user_agent
 514
 515     # Set referer
 516     if opts.referer is not None:
 517         std_headers['Referer'] = opts.referer
 518
 519     # Dump user agent
 520     if opts.dump_user_agent:
 521         compat_print(std_headers['User-Agent'])
 522         sys.exit(0)
 523
 524     # Batch file verification
 525     batchurls = []
 526     if opts.batchfile is not None:
 527         try:
 528             if opts.batchfile == '-':
 529                 batchfd = sys.stdin
 530             else:
 531                 batchfd = open(opts.batchfile, 'r')
 532             batchurls = batchfd.readlines()
 533             batchurls = [x.strip() for x in batchurls]
 534             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 535             if opts.verbose:
 536                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 537         except IOError:
 538             sys.exit(u'ERROR: batch file could not be read')
 539     all_urls = batchurls + args
 540     all_urls = [url.strip() for url in all_urls]
 541     _enc = preferredencoding()
 542     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 543
 544     extractors = gen_extractors()
 545
 546     if opts.list_extractors:
 547         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 548             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 549             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 550             for mu in matchedUrls:
 551                 compat_print(u'  ' + mu)
 552         sys.exit(0)
 553     if opts.list_extractor_descriptions:
 554         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 555             if not ie._WORKING:
 556                 continue
 557             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 558             if desc is False:
 559                 continue
 560             if hasattr(ie, 'SEARCH_KEY'):
 561                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 562                 _COUNTS = (u'', u'5', u'10', u'all')
 563                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 564             compat_print(desc)
 565         sys.exit(0)
 566
 567
 568     # Conflicting, missing and erroneous options
 569     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 570         parser.error(u'using .netrc conflicts with giving username/password')
 571     if opts.password is not None and opts.username is None:
 572         parser.error(u'account username missing\n')
 573     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 574         parser.error(u'using output template conflicts with using title, video ID or auto number')
 575     if opts.usetitle and opts.useid:
 576         parser.error(u'using title conflicts with using video ID')
 577     if opts.username is not None and opts.password is None:
 578         opts.password = getpass.getpass(u'Type account password and press return:')
 579     if opts.ratelimit is not None:
 580         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 581         if numeric_limit is None:
 582             parser.error(u'invalid rate limit specified')
 583         opts.ratelimit = numeric_limit
 584     if opts.min_filesize is not None:
 585         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 586         if numeric_limit is None:
 587             parser.error(u'invalid min_filesize specified')
 588         opts.min_filesize = numeric_limit
 589     if opts.max_filesize is not None:
 590         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 591         if numeric_limit is None:
 592             parser.error(u'invalid max_filesize specified')
 593         opts.max_filesize = numeric_limit
 594     if opts.retries is not None:
 595         try:
 596             opts.retries = int(opts.retries)
 597         except (TypeError, ValueError):
 598             parser.error(u'invalid retry count specified')
 599     if opts.buffersize is not None:
 600         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 601         if numeric_buffersize is None:
 602             parser.error(u'invalid buffer size specified')
 603         opts.buffersize = numeric_buffersize
 604     if opts.playliststart <= 0:
 605         raise ValueError(u'Playlist start must be positive')
 606     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 607         raise ValueError(u'Playlist end must be greater than playlist start')
 608     if opts.extractaudio:
 609         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 610             parser.error(u'invalid audio format specified')
 611     if opts.audioquality:
 612         opts.audioquality = opts.audioquality.strip('k').strip('K')
 613         if not opts.audioquality.isdigit():
 614             parser.error(u'invalid audio quality specified')
 615     if opts.recodevideo is not None:
 616         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 617             parser.error(u'invalid video recode format specified')
 618     if opts.date is not None:
 619         date = DateRange.day(opts.date)
 620     else:
 621         date = DateRange(opts.dateafter, opts.datebefore)
 622
 623     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 624     # this was the old behaviour if only --all-sub was given.
 625     if opts.allsubtitles and (opts.writeautomaticsub == False):
 626         opts.writesubtitles = True
 627
 628     if sys.version_info < (3,):
 629         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 630         if opts.outtmpl is not None:
 631             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 632     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 633             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 634             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 635             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 636             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 637             or (opts.useid and u'%(id)s.%(ext)s')
 638             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 639             or u'%(title)s-%(id)s.%(ext)s')
 640     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 641         parser.error(u'Cannot download a video and extract audio into the same'
 642                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 643                      u' template'.format(outtmpl))
 644
 645     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 646     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 647
 648     ydl_opts = {
 649         'usenetrc': opts.usenetrc,
 650         'username': opts.username,
 651         'password': opts.password,
 652         'videopassword': opts.videopassword,
 653         'quiet': (opts.quiet or any_printing),
 654         'forceurl': opts.geturl,
 655         'forcetitle': opts.gettitle,
 656         'forceid': opts.getid,
 657         'forcethumbnail': opts.getthumbnail,
 658         'forcedescription': opts.getdescription,
 659         'forceduration': opts.getduration,
 660         'forcefilename': opts.getfilename,
 661         'forceformat': opts.getformat,
 662         'forcejson': opts.dumpjson,
 663         'simulate': opts.simulate,
 664         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 665         'format': opts.format,
 666         'format_limit': opts.format_limit,
 667         'listformats': opts.listformats,
 668         'outtmpl': outtmpl,
 669         'autonumber_size': opts.autonumber_size,
 670         'restrictfilenames': opts.restrictfilenames,
 671         'ignoreerrors': opts.ignoreerrors,
 672         'ratelimit': opts.ratelimit,
 673         'nooverwrites': opts.nooverwrites,
 674         'retries': opts.retries,
 675         'buffersize': opts.buffersize,
 676         'noresizebuffer': opts.noresizebuffer,
 677         'continuedl': opts.continue_dl,
 678         'noprogress': opts.noprogress,
 679         'progress_with_newline': opts.progress_with_newline,
 680         'playliststart': opts.playliststart,
 681         'playlistend': opts.playlistend,
 682         'noplaylist': opts.noplaylist,
 683         'logtostderr': opts.outtmpl == '-',
 684         'consoletitle': opts.consoletitle,
 685         'nopart': opts.nopart,
 686         'updatetime': opts.updatetime,
 687         'writedescription': opts.writedescription,
 688         'writeannotations': opts.writeannotations,
 689         'writeinfojson': opts.writeinfojson,
 690         'writethumbnail': opts.writethumbnail,
 691         'writesubtitles': opts.writesubtitles,
 692         'writeautomaticsub': opts.writeautomaticsub,
 693         'allsubtitles': opts.allsubtitles,
 694         'listsubtitles': opts.listsubtitles,
 695         'subtitlesformat': opts.subtitlesformat,
 696         'subtitleslangs': opts.subtitleslangs,
 697         'matchtitle': decodeOption(opts.matchtitle),
 698         'rejecttitle': decodeOption(opts.rejecttitle),
 699         'max_downloads': opts.max_downloads,
 700         'prefer_free_formats': opts.prefer_free_formats,
 701         'verbose': opts.verbose,
 702         'dump_intermediate_pages': opts.dump_intermediate_pages,
 703         'write_pages': opts.write_pages,
 704         'test': opts.test,
 705         'keepvideo': opts.keepvideo,
 706         'min_filesize': opts.min_filesize,
 707         'max_filesize': opts.max_filesize,
 708         'min_views': opts.min_views,
 709         'max_views': opts.max_views,
 710         'daterange': date,
 711         'cachedir': opts.cachedir,
 712         'youtube_print_sig_code': opts.youtube_print_sig_code,
 713         'age_limit': opts.age_limit,
 714         'download_archive': download_archive_fn,
 715         'cookiefile': opts.cookiefile,
 716         'nocheckcertificate': opts.no_check_certificate,
 717         'proxy': opts.proxy,
 718         'socket_timeout': opts.socket_timeout,
 719         'bidi_workaround': opts.bidi_workaround,
 720         'debug_printtraffic': opts.debug_printtraffic,
 721         'prefer_ffmpeg': opts.prefer_ffmpeg,
 722         'include_ads': opts.include_ads,
 723     }
 724
 725     with YoutubeDL(ydl_opts) as ydl:
 726         ydl.print_debug_header()
 727         ydl.add_default_info_extractors()
 728
 729         # PostProcessors
 730         # Add the metadata pp first, the other pps will copy it
 731         if opts.addmetadata:
 732             ydl.add_post_processor(FFmpegMetadataPP())
 733         if opts.extractaudio:
 734             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 735         if opts.recodevideo:
 736             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 737         if opts.embedsubtitles:
 738             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 739         if opts.xattrs:
 740             ydl.add_post_processor(XAttrMetadataPP())
 741
 742         # Update version
 743         if opts.update_self:
 744             update_self(ydl.to_screen, opts.verbose)
 745
 746         # Maybe do nothing
 747         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 748             if not opts.update_self:
 749                 parser.error(u'you must provide at least one URL')
 750             else:
 751                 sys.exit()
 752
 753         try:
 754             if opts.load_info_filename is not None:
 755                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 756             else:
 757                 retcode = ydl.download(all_urls)
 758         except MaxDownloadsReached:
 759             ydl.to_screen(u'--max-download limit reached, aborting.')
 760             retcode = 101
 761
 762     sys.exit(retcode)
 763
 764
 765 def main(argv=None):
 766     try:
 767         _real_main(argv)
 768     except DownloadError:
 769         sys.exit(1)
 770     except SameFileError:
 771         sys.exit(u'ERROR: fixed output name but more than one file to download')
 772     except KeyboardInterrupt:
 773         sys.exit(u'\nERROR: Interrupted by user')