youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50 )
  51
  52 __license__ = 'Public Domain'
  53
  54 import codecs
  55 import getpass
  56 import locale
  57 import optparse
  58 import os
  59 import random
  60 import re
  61 import shlex
  62 import sys
  63
  64
  65 from .utils import (
  66     compat_print,
  67     DateRange,
  68     decodeOption,
  69     get_term_width,
  70     DownloadError,
  71     get_cachedir,
  72     MaxDownloadsReached,
  73     preferredencoding,
  74     read_batch_urls,
  75     SameFileError,
  76     setproctitle,
  77     std_headers,
  78     write_string,
  79 )
  80 from .update import update_self
  81 from .FileDownloader import (
  82     FileDownloader,
  83 )
  84 from .extractor import gen_extractors
  85 from .version import __version__
  86 from .YoutubeDL import YoutubeDL
  87 from .postprocessor import (
  88     FFmpegMetadataPP,
  89     FFmpegVideoConvertor,
  90     FFmpegExtractAudioPP,
  91     FFmpegEmbedSubtitlePP,
  92     XAttrMetadataPP,
  93 )
  94
  95
  96 def parseOpts(overrideArguments=None):
  97     def _readOptions(filename_bytes, default=[]):
  98         try:
  99             optionf = open(filename_bytes)
 100         except IOError:
 101             return default  # silently skip if file is not present
 102         try:
 103             res = []
 104             for l in optionf:
 105                 res += shlex.split(l, comments=True)
 106         finally:
 107             optionf.close()
 108         return res
 109
 110     def _readUserConf():
 111         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 112         if xdg_config_home:
 113             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 114             if not os.path.isfile(userConfFile):
 115                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 116         else:
 117             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 118             if not os.path.isfile(userConfFile):
 119                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 120         userConf = _readOptions(userConfFile, None)
 121
 122         if userConf is None:
 123             appdata_dir = os.environ.get('appdata')
 124             if appdata_dir:
 125                 userConf = _readOptions(
 126                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 127                     default=None)
 128                 if userConf is None:
 129                     userConf = _readOptions(
 130                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 131                         default=None)
 132
 133         if userConf is None:
 134             userConf = _readOptions(
 135                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 136                 default=None)
 137         if userConf is None:
 138             userConf = _readOptions(
 139                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 140                 default=None)
 141
 142         if userConf is None:
 143             userConf = []
 144
 145         return userConf
 146
 147     def _format_option_string(option):
 148         ''' ('-o', '--option') -> -o, --format METAVAR'''
 149
 150         opts = []
 151
 152         if option._short_opts:
 153             opts.append(option._short_opts[0])
 154         if option._long_opts:
 155             opts.append(option._long_opts[0])
 156         if len(opts) > 1:
 157             opts.insert(1, ', ')
 158
 159         if option.takes_value(): opts.append(' %s' % option.metavar)
 160
 161         return "".join(opts)
 162
 163     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 164         setattr(parser.values, option.dest, value.split(','))
 165
 166     def _hide_login_info(opts):
 167         opts = list(opts)
 168         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 169             try:
 170                 i = opts.index(private_opt)
 171                 opts[i+1] = '<PRIVATE>'
 172             except ValueError:
 173                 pass
 174         return opts
 175
 176     max_width = 80
 177     max_help_position = 80
 178
 179     # No need to wrap help messages if we're on a wide console
 180     columns = get_term_width()
 181     if columns: max_width = columns
 182
 183     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 184     fmt.format_option_strings = _format_option_string
 185
 186     kw = {
 187         'version'   : __version__,
 188         'formatter' : fmt,
 189         'usage' : '%prog [options] url [url...]',
 190         'conflict_handler' : 'resolve',
 191     }
 192
 193     parser = optparse.OptionParser(**kw)
 194
 195     # option groups
 196     general        = optparse.OptionGroup(parser, 'General Options')
 197     selection      = optparse.OptionGroup(parser, 'Video Selection')
 198     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 199     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 200     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 201     downloader     = optparse.OptionGroup(parser, 'Download Options')
 202     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 203     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 204     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 205
 206     general.add_option('-h', '--help',
 207             action='help', help='print this help text and exit')
 208     general.add_option('-v', '--version',
 209             action='version', help='print program version and exit')
 210     general.add_option('-U', '--update',
 211             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 212     general.add_option('-i', '--ignore-errors',
 213             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 214     general.add_option('--abort-on-error',
 215             action='store_false', dest='ignoreerrors',
 216             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 217     general.add_option('--dump-user-agent',
 218             action='store_true', dest='dump_user_agent',
 219             help='display the current browser identification', default=False)
 220     general.add_option('--user-agent',
 221             dest='user_agent', help='specify a custom user agent', metavar='UA')
 222     general.add_option('--referer',
 223             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 224             metavar='REF', default=None)
 225     general.add_option('--list-extractors',
 226             action='store_true', dest='list_extractors',
 227             help='List all supported extractors and the URLs they would handle', default=False)
 228     general.add_option('--extractor-descriptions',
 229             action='store_true', dest='list_extractor_descriptions',
 230             help='Output descriptions of all supported extractors', default=False)
 231     general.add_option(
 232         '--proxy', dest='proxy', default=None, metavar='URL',
 233         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 234     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 235     general.add_option(
 236         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 237         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 238     general.add_option(
 239         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 240         help='Disable filesystem caching')
 241     general.add_option(
 242         '--socket-timeout', dest='socket_timeout',
 243         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 244     general.add_option(
 245         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 246         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 247     general.add_option('--default-search',
 248             dest='default_search', metavar='PREFIX',
 249             help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". By default (with value "auto") youtube-dl guesses.')
 250     general.add_option(
 251         '--ignore-config',
 252         action='store_true',
 253         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 254
 255
 256     selection.add_option(
 257         '--playlist-start',
 258         dest='playliststart', metavar='NUMBER', default=1, type=int,
 259         help='playlist video to start at (default is %default)')
 260     selection.add_option(
 261         '--playlist-end',
 262         dest='playlistend', metavar='NUMBER', default=None, type=int,
 263         help='playlist video to end at (default is last)')
 264     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 265     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 266     selection.add_option('--max-downloads', metavar='NUMBER',
 267                          dest='max_downloads', type=int, default=None,
 268                          help='Abort after downloading NUMBER files')
 269     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 270     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 271     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 272     selection.add_option(
 273         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 274         help='download only videos uploaded on or before this date (i.e. inclusive)')
 275     selection.add_option(
 276         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 277         help='download only videos uploaded on or after this date (i.e. inclusive)')
 278     selection.add_option(
 279         '--min-views', metavar='COUNT', dest='min_views',
 280         default=None, type=int,
 281         help="Do not download any videos with less than COUNT views",)
 282     selection.add_option(
 283         '--max-views', metavar='COUNT', dest='max_views',
 284         default=None, type=int,
 285         help="Do not download any videos with more than COUNT views",)
 286     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 287     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 288                          help='download only videos suitable for the given age',
 289                          default=None, type=int)
 290     selection.add_option('--download-archive', metavar='FILE',
 291                          dest='download_archive',
 292                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 293     selection.add_option(
 294         '--include-ads', dest='include_ads',
 295         action='store_true',
 296         help='Download advertisements as well (experimental)')
 297     selection.add_option(
 298         '--youtube-include-dash-manifest', action='store_true',
 299         dest='youtube_include_dash_manifest', default=False,
 300         help='Try to download the DASH manifest on YouTube videos (experimental)')
 301
 302     authentication.add_option('-u', '--username',
 303             dest='username', metavar='USERNAME', help='account username')
 304     authentication.add_option('-p', '--password',
 305             dest='password', metavar='PASSWORD', help='account password')
 306     authentication.add_option('-n', '--netrc',
 307             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 308     authentication.add_option('--video-password',
 309             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 310
 311
 312     video_format.add_option('-f', '--format',
 313             action='store', dest='format', metavar='FORMAT', default=None,
 314             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestaudio", "worst", and "worstaudio". By default, youtube-dl will pick the best quality.')
 315     video_format.add_option('--all-formats',
 316             action='store_const', dest='format', help='download all available video formats', const='all')
 317     video_format.add_option('--prefer-free-formats',
 318             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 319     video_format.add_option('--max-quality',
 320             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 321     video_format.add_option('-F', '--list-formats',
 322             action='store_true', dest='listformats', help='list all available formats')
 323
 324     subtitles.add_option('--write-sub', '--write-srt',
 325             action='store_true', dest='writesubtitles',
 326             help='write subtitle file', default=False)
 327     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 328             action='store_true', dest='writeautomaticsub',
 329             help='write automatic subtitle file (youtube only)', default=False)
 330     subtitles.add_option('--all-subs',
 331             action='store_true', dest='allsubtitles',
 332             help='downloads all the available subtitles of the video', default=False)
 333     subtitles.add_option('--list-subs',
 334             action='store_true', dest='listsubtitles',
 335             help='lists all available subtitles for the video', default=False)
 336     subtitles.add_option('--sub-format',
 337             action='store', dest='subtitlesformat', metavar='FORMAT',
 338             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 339     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 340             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 341             default=[], callback=_comma_separated_values_options_callback,
 342             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 343
 344     downloader.add_option('-r', '--rate-limit',
 345             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 346     downloader.add_option('-R', '--retries',
 347             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 348     downloader.add_option('--buffer-size',
 349             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 350     downloader.add_option('--no-resize-buffer',
 351             action='store_true', dest='noresizebuffer',
 352             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 353     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 354
 355     verbosity.add_option('-q', '--quiet',
 356             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 357     verbosity.add_option('-s', '--simulate',
 358             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 359     verbosity.add_option('--skip-download',
 360             action='store_true', dest='skip_download', help='do not download the video', default=False)
 361     verbosity.add_option('-g', '--get-url',
 362             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 363     verbosity.add_option('-e', '--get-title',
 364             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 365     verbosity.add_option('--get-id',
 366             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 367     verbosity.add_option('--get-thumbnail',
 368             action='store_true', dest='getthumbnail',
 369             help='simulate, quiet but print thumbnail URL', default=False)
 370     verbosity.add_option('--get-description',
 371             action='store_true', dest='getdescription',
 372             help='simulate, quiet but print video description', default=False)
 373     verbosity.add_option('--get-duration',
 374             action='store_true', dest='getduration',
 375             help='simulate, quiet but print video length', default=False)
 376     verbosity.add_option('--get-filename',
 377             action='store_true', dest='getfilename',
 378             help='simulate, quiet but print output filename', default=False)
 379     verbosity.add_option('--get-format',
 380             action='store_true', dest='getformat',
 381             help='simulate, quiet but print output format', default=False)
 382     verbosity.add_option('-j', '--dump-json',
 383             action='store_true', dest='dumpjson',
 384             help='simulate, quiet but print JSON information', default=False)
 385     verbosity.add_option('--newline',
 386             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 387     verbosity.add_option('--no-progress',
 388             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 389     verbosity.add_option('--console-title',
 390             action='store_true', dest='consoletitle',
 391             help='display progress in console titlebar', default=False)
 392     verbosity.add_option('-v', '--verbose',
 393             action='store_true', dest='verbose', help='print various debugging information', default=False)
 394     verbosity.add_option('--dump-intermediate-pages',
 395             action='store_true', dest='dump_intermediate_pages', default=False,
 396             help='print downloaded pages to debug problems (very verbose)')
 397     verbosity.add_option('--write-pages',
 398             action='store_true', dest='write_pages', default=False,
 399             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 400     verbosity.add_option('--youtube-print-sig-code',
 401             action='store_true', dest='youtube_print_sig_code', default=False,
 402             help=optparse.SUPPRESS_HELP)
 403     verbosity.add_option('--print-traffic',
 404             dest='debug_printtraffic', action='store_true', default=False,
 405             help='Display sent and read HTTP traffic')
 406
 407
 408     filesystem.add_option('-t', '--title',
 409             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 410     filesystem.add_option('--id',
 411             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 412     filesystem.add_option('-l', '--literal',
 413             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 414     filesystem.add_option('-A', '--auto-number',
 415             action='store_true', dest='autonumber',
 416             help='number downloaded files starting from 00000', default=False)
 417     filesystem.add_option('-o', '--output',
 418             dest='outtmpl', metavar='TEMPLATE',
 419             help=('output filename template. Use %(title)s to get the title, '
 420                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 421                   '%(autonumber)s to get an automatically incremented number, '
 422                   '%(ext)s for the filename extension, '
 423                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 424                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 425                   '%(upload_date)s for the upload date (YYYYMMDD), '
 426                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 427                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 428                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 429                   'Use - to output to stdout. Can also be used to download to a different directory, '
 430                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 431     filesystem.add_option('--autonumber-size',
 432             dest='autonumber_size', metavar='NUMBER',
 433             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 434     filesystem.add_option('--restrict-filenames',
 435             action='store_true', dest='restrictfilenames',
 436             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 437     filesystem.add_option('-a', '--batch-file',
 438             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 439     filesystem.add_option('--load-info',
 440             dest='load_info_filename', metavar='FILE',
 441             help='json file containing the video information (created with the "--write-json" option)')
 442     filesystem.add_option('-w', '--no-overwrites',
 443             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 444     filesystem.add_option('-c', '--continue',
 445             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 446     filesystem.add_option('--no-continue',
 447             action='store_false', dest='continue_dl',
 448             help='do not resume partially downloaded files (restart from beginning)')
 449     filesystem.add_option('--cookies',
 450             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 451     filesystem.add_option('--no-part',
 452             action='store_true', dest='nopart', help='do not use .part files', default=False)
 453     filesystem.add_option('--no-mtime',
 454             action='store_false', dest='updatetime',
 455             help='do not use the Last-modified header to set the file modification time', default=True)
 456     filesystem.add_option('--write-description',
 457             action='store_true', dest='writedescription',
 458             help='write video description to a .description file', default=False)
 459     filesystem.add_option('--write-info-json',
 460             action='store_true', dest='writeinfojson',
 461             help='write video metadata to a .info.json file', default=False)
 462     filesystem.add_option('--write-annotations',
 463             action='store_true', dest='writeannotations',
 464             help='write video annotations to a .annotation file', default=False)
 465     filesystem.add_option('--write-thumbnail',
 466             action='store_true', dest='writethumbnail',
 467             help='write thumbnail image to disk', default=False)
 468
 469
 470     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 471             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 472     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 473             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 474     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 475             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 476     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 477             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 478     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 479             help='keeps the video file on disk after the post-processing; the video is erased by default')
 480     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 481             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 482     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 483             help='embed subtitles in the video (only for mp4 videos)')
 484     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 485             help='write metadata to the video file')
 486     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 487             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 488     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 489         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 490     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 491         help='Prefer ffmpeg over avconv for running the postprocessors')
 492
 493
 494     parser.add_option_group(general)
 495     parser.add_option_group(selection)
 496     parser.add_option_group(downloader)
 497     parser.add_option_group(filesystem)
 498     parser.add_option_group(verbosity)
 499     parser.add_option_group(video_format)
 500     parser.add_option_group(subtitles)
 501     parser.add_option_group(authentication)
 502     parser.add_option_group(postproc)
 503
 504     if overrideArguments is not None:
 505         opts, args = parser.parse_args(overrideArguments)
 506         if opts.verbose:
 507             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 508     else:
 509         commandLineConf = sys.argv[1:]
 510         if '--ignore-config' in commandLineConf:
 511             systemConf = []
 512             userConf = []
 513         else:
 514             systemConf = _readOptions('/etc/youtube-dl.conf')
 515             if '--ignore-config' in systemConf:
 516                 userConf = []
 517             else:
 518                 userConf = _readUserConf()
 519         argv = systemConf + userConf + commandLineConf
 520
 521         opts, args = parser.parse_args(argv)
 522         if opts.verbose:
 523             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 524             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 525             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 526             write_string(u'[debug] Encodings: locale %r, fs %r, out %r, pref: %r\n' %
 527                          (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, preferredencoding()))
 528
 529     return parser, opts, args
 530
 531
 532 def _real_main(argv=None):
 533     # Compatibility fixes for Windows
 534     if sys.platform == 'win32':
 535         # https://github.com/rg3/youtube-dl/issues/820
 536         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 537
 538     setproctitle(u'youtube-dl')
 539
 540     parser, opts, args = parseOpts(argv)
 541
 542     # Set user agent
 543     if opts.user_agent is not None:
 544         std_headers['User-Agent'] = opts.user_agent
 545
 546     # Set referer
 547     if opts.referer is not None:
 548         std_headers['Referer'] = opts.referer
 549
 550     # Dump user agent
 551     if opts.dump_user_agent:
 552         compat_print(std_headers['User-Agent'])
 553         sys.exit(0)
 554
 555     # Batch file verification
 556     batch_urls = []
 557     if opts.batchfile is not None:
 558         try:
 559             if opts.batchfile == '-':
 560                 batchfd = sys.stdin
 561             else:
 562                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 563             batch_urls = read_batch_urls(batchfd)
 564             if opts.verbose:
 565                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 566         except IOError:
 567             sys.exit(u'ERROR: batch file could not be read')
 568     all_urls = batch_urls + args
 569     all_urls = [url.strip() for url in all_urls]
 570     _enc = preferredencoding()
 571     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 572
 573     extractors = gen_extractors()
 574
 575     if opts.list_extractors:
 576         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 577             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 578             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 579             for mu in matchedUrls:
 580                 compat_print(u'  ' + mu)
 581         sys.exit(0)
 582     if opts.list_extractor_descriptions:
 583         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 584             if not ie._WORKING:
 585                 continue
 586             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 587             if desc is False:
 588                 continue
 589             if hasattr(ie, 'SEARCH_KEY'):
 590                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 591                 _COUNTS = (u'', u'5', u'10', u'all')
 592                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 593             compat_print(desc)
 594         sys.exit(0)
 595
 596
 597     # Conflicting, missing and erroneous options
 598     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 599         parser.error(u'using .netrc conflicts with giving username/password')
 600     if opts.password is not None and opts.username is None:
 601         parser.error(u'account username missing\n')
 602     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 603         parser.error(u'using output template conflicts with using title, video ID or auto number')
 604     if opts.usetitle and opts.useid:
 605         parser.error(u'using title conflicts with using video ID')
 606     if opts.username is not None and opts.password is None:
 607         opts.password = getpass.getpass(u'Type account password and press return:')
 608     if opts.ratelimit is not None:
 609         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 610         if numeric_limit is None:
 611             parser.error(u'invalid rate limit specified')
 612         opts.ratelimit = numeric_limit
 613     if opts.min_filesize is not None:
 614         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 615         if numeric_limit is None:
 616             parser.error(u'invalid min_filesize specified')
 617         opts.min_filesize = numeric_limit
 618     if opts.max_filesize is not None:
 619         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 620         if numeric_limit is None:
 621             parser.error(u'invalid max_filesize specified')
 622         opts.max_filesize = numeric_limit
 623     if opts.retries is not None:
 624         try:
 625             opts.retries = int(opts.retries)
 626         except (TypeError, ValueError):
 627             parser.error(u'invalid retry count specified')
 628     if opts.buffersize is not None:
 629         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 630         if numeric_buffersize is None:
 631             parser.error(u'invalid buffer size specified')
 632         opts.buffersize = numeric_buffersize
 633     if opts.playliststart <= 0:
 634         raise ValueError(u'Playlist start must be positive')
 635     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 636         raise ValueError(u'Playlist end must be greater than playlist start')
 637     if opts.extractaudio:
 638         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 639             parser.error(u'invalid audio format specified')
 640     if opts.audioquality:
 641         opts.audioquality = opts.audioquality.strip('k').strip('K')
 642         if not opts.audioquality.isdigit():
 643             parser.error(u'invalid audio quality specified')
 644     if opts.recodevideo is not None:
 645         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 646             parser.error(u'invalid video recode format specified')
 647     if opts.date is not None:
 648         date = DateRange.day(opts.date)
 649     else:
 650         date = DateRange(opts.dateafter, opts.datebefore)
 651     if opts.default_search not in ('auto', None) and ':' not in opts.default_search:
 652         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 653
 654     # Do not download videos when there are audio-only formats
 655     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 656         opts.format = 'bestaudio/best'
 657
 658     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 659     # this was the old behaviour if only --all-sub was given.
 660     if opts.allsubtitles and (opts.writeautomaticsub == False):
 661         opts.writesubtitles = True
 662
 663     if sys.version_info < (3,):
 664         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 665         if opts.outtmpl is not None:
 666             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 667     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 668             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 669             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 670             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 671             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 672             or (opts.useid and u'%(id)s.%(ext)s')
 673             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 674             or u'%(title)s-%(id)s.%(ext)s')
 675     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 676         parser.error(u'Cannot download a video and extract audio into the same'
 677                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 678                      u' template'.format(outtmpl))
 679
 680     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 681     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 682
 683     ydl_opts = {
 684         'usenetrc': opts.usenetrc,
 685         'username': opts.username,
 686         'password': opts.password,
 687         'videopassword': opts.videopassword,
 688         'quiet': (opts.quiet or any_printing),
 689         'forceurl': opts.geturl,
 690         'forcetitle': opts.gettitle,
 691         'forceid': opts.getid,
 692         'forcethumbnail': opts.getthumbnail,
 693         'forcedescription': opts.getdescription,
 694         'forceduration': opts.getduration,
 695         'forcefilename': opts.getfilename,
 696         'forceformat': opts.getformat,
 697         'forcejson': opts.dumpjson,
 698         'simulate': opts.simulate,
 699         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 700         'format': opts.format,
 701         'format_limit': opts.format_limit,
 702         'listformats': opts.listformats,
 703         'outtmpl': outtmpl,
 704         'autonumber_size': opts.autonumber_size,
 705         'restrictfilenames': opts.restrictfilenames,
 706         'ignoreerrors': opts.ignoreerrors,
 707         'ratelimit': opts.ratelimit,
 708         'nooverwrites': opts.nooverwrites,
 709         'retries': opts.retries,
 710         'buffersize': opts.buffersize,
 711         'noresizebuffer': opts.noresizebuffer,
 712         'continuedl': opts.continue_dl,
 713         'noprogress': opts.noprogress,
 714         'progress_with_newline': opts.progress_with_newline,
 715         'playliststart': opts.playliststart,
 716         'playlistend': opts.playlistend,
 717         'noplaylist': opts.noplaylist,
 718         'logtostderr': opts.outtmpl == '-',
 719         'consoletitle': opts.consoletitle,
 720         'nopart': opts.nopart,
 721         'updatetime': opts.updatetime,
 722         'writedescription': opts.writedescription,
 723         'writeannotations': opts.writeannotations,
 724         'writeinfojson': opts.writeinfojson,
 725         'writethumbnail': opts.writethumbnail,
 726         'writesubtitles': opts.writesubtitles,
 727         'writeautomaticsub': opts.writeautomaticsub,
 728         'allsubtitles': opts.allsubtitles,
 729         'listsubtitles': opts.listsubtitles,
 730         'subtitlesformat': opts.subtitlesformat,
 731         'subtitleslangs': opts.subtitleslangs,
 732         'matchtitle': decodeOption(opts.matchtitle),
 733         'rejecttitle': decodeOption(opts.rejecttitle),
 734         'max_downloads': opts.max_downloads,
 735         'prefer_free_formats': opts.prefer_free_formats,
 736         'verbose': opts.verbose,
 737         'dump_intermediate_pages': opts.dump_intermediate_pages,
 738         'write_pages': opts.write_pages,
 739         'test': opts.test,
 740         'keepvideo': opts.keepvideo,
 741         'min_filesize': opts.min_filesize,
 742         'max_filesize': opts.max_filesize,
 743         'min_views': opts.min_views,
 744         'max_views': opts.max_views,
 745         'daterange': date,
 746         'cachedir': opts.cachedir,
 747         'youtube_print_sig_code': opts.youtube_print_sig_code,
 748         'age_limit': opts.age_limit,
 749         'download_archive': download_archive_fn,
 750         'cookiefile': opts.cookiefile,
 751         'nocheckcertificate': opts.no_check_certificate,
 752         'proxy': opts.proxy,
 753         'socket_timeout': opts.socket_timeout,
 754         'bidi_workaround': opts.bidi_workaround,
 755         'debug_printtraffic': opts.debug_printtraffic,
 756         'prefer_ffmpeg': opts.prefer_ffmpeg,
 757         'include_ads': opts.include_ads,
 758         'default_search': opts.default_search,
 759         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 760     }
 761
 762     with YoutubeDL(ydl_opts) as ydl:
 763         ydl.print_debug_header()
 764         ydl.add_default_info_extractors()
 765
 766         # PostProcessors
 767         # Add the metadata pp first, the other pps will copy it
 768         if opts.addmetadata:
 769             ydl.add_post_processor(FFmpegMetadataPP())
 770         if opts.extractaudio:
 771             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 772         if opts.recodevideo:
 773             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 774         if opts.embedsubtitles:
 775             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 776         if opts.xattrs:
 777             ydl.add_post_processor(XAttrMetadataPP())
 778
 779         # Update version
 780         if opts.update_self:
 781             update_self(ydl.to_screen, opts.verbose)
 782
 783         # Maybe do nothing
 784         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 785             if not opts.update_self:
 786                 parser.error(u'you must provide at least one URL')
 787             else:
 788                 sys.exit()
 789
 790         try:
 791             if opts.load_info_filename is not None:
 792                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 793             else:
 794                 retcode = ydl.download(all_urls)
 795         except MaxDownloadsReached:
 796             ydl.to_screen(u'--max-download limit reached, aborting.')
 797             retcode = 101
 798
 799     sys.exit(retcode)
 800
 801
 802 def main(argv=None):
 803     try:
 804         _real_main(argv)
 805     except DownloadError:
 806         sys.exit(1)
 807     except SameFileError:
 808         sys.exit(u'ERROR: fixed output name but more than one file to download')
 809     except KeyboardInterrupt:
 810         sys.exit(u'\nERROR: Interrupted by user')