youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37 )
  38
  39 __license__ = 'Public Domain'
  40
  41 import codecs
  42 import collections
  43 import getpass
  44 import optparse
  45 import os
  46 import random
  47 import re
  48 import shlex
  49 import socket
  50 import subprocess
  51 import sys
  52 import traceback
  53 import platform
  54
  55
  56 from .utils import (
  57     compat_cookiejar,
  58     compat_print,
  59     compat_str,
  60     compat_urllib_request,
  61     DateRange,
  62     decodeOption,
  63     determine_ext,
  64     DownloadError,
  65     get_cachedir,
  66     make_HTTPS_handler,
  67     MaxDownloadsReached,
  68     platform_name,
  69     preferredencoding,
  70     SameFileError,
  71     std_headers,
  72     write_string,
  73     YoutubeDLHandler,
  74 )
  75 from .update import update_self
  76 from .version import __version__
  77 from .FileDownloader import (
  78     FileDownloader,
  79 )
  80 from .extractor import gen_extractors
  81 from .YoutubeDL import YoutubeDL
  82 from .PostProcessor import (
  83     FFmpegMetadataPP,
  84     FFmpegVideoConvertor,
  85     FFmpegExtractAudioPP,
  86     FFmpegEmbedSubtitlePP,
  87 )
  88
  89
  90 def parseOpts(overrideArguments=None):
  91     def _readOptions(filename_bytes):
  92         try:
  93             optionf = open(filename_bytes)
  94         except IOError:
  95             return [] # silently skip if file is not present
  96         try:
  97             res = []
  98             for l in optionf:
  99                 res += shlex.split(l, comments=True)
 100         finally:
 101             optionf.close()
 102         return res
 103
 104     def _format_option_string(option):
 105         ''' ('-o', '--option') -> -o, --format METAVAR'''
 106
 107         opts = []
 108
 109         if option._short_opts:
 110             opts.append(option._short_opts[0])
 111         if option._long_opts:
 112             opts.append(option._long_opts[0])
 113         if len(opts) > 1:
 114             opts.insert(1, ', ')
 115
 116         if option.takes_value(): opts.append(' %s' % option.metavar)
 117
 118         return "".join(opts)
 119
 120     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 121         setattr(parser.values, option.dest, value.split(','))
 122
 123     def _find_term_columns():
 124         columns = os.environ.get('COLUMNS', None)
 125         if columns:
 126             return int(columns)
 127
 128         try:
 129             sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 130             out,err = sp.communicate()
 131             return int(out.split()[1])
 132         except:
 133             pass
 134         return None
 135
 136     def _hide_login_info(opts):
 137         opts = list(opts)
 138         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 139             try:
 140                 i = opts.index(private_opt)
 141                 opts[i+1] = '<PRIVATE>'
 142             except ValueError:
 143                 pass
 144         return opts
 145
 146     max_width = 80
 147     max_help_position = 80
 148
 149     # No need to wrap help messages if we're on a wide console
 150     columns = _find_term_columns()
 151     if columns: max_width = columns
 152
 153     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 154     fmt.format_option_strings = _format_option_string
 155
 156     kw = {
 157         'version'   : __version__,
 158         'formatter' : fmt,
 159         'usage' : '%prog [options] url [url...]',
 160         'conflict_handler' : 'resolve',
 161     }
 162
 163     parser = optparse.OptionParser(**kw)
 164
 165     # option groups
 166     general        = optparse.OptionGroup(parser, 'General Options')
 167     selection      = optparse.OptionGroup(parser, 'Video Selection')
 168     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 169     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 170     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 171     downloader     = optparse.OptionGroup(parser, 'Download Options')
 172     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 173     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 174     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 175
 176     general.add_option('-h', '--help',
 177             action='help', help='print this help text and exit')
 178     general.add_option('-v', '--version',
 179             action='version', help='print program version and exit')
 180     general.add_option('-U', '--update',
 181             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 182     general.add_option('-i', '--ignore-errors',
 183             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 184     general.add_option('--abort-on-error',
 185             action='store_false', dest='ignoreerrors',
 186             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 187     general.add_option('--dump-user-agent',
 188             action='store_true', dest='dump_user_agent',
 189             help='display the current browser identification', default=False)
 190     general.add_option('--user-agent',
 191             dest='user_agent', help='specify a custom user agent', metavar='UA')
 192     general.add_option('--referer',
 193             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 194             metavar='REF', default=None)
 195     general.add_option('--list-extractors',
 196             action='store_true', dest='list_extractors',
 197             help='List all supported extractors and the URLs they would handle', default=False)
 198     general.add_option('--extractor-descriptions',
 199             action='store_true', dest='list_extractor_descriptions',
 200             help='Output descriptions of all supported extractors', default=False)
 201     general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
 202     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 203     general.add_option(
 204         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 205         help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
 206     general.add_option(
 207         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 208         help='Disable filesystem caching')
 209
 210
 211     selection.add_option('--playlist-start',
 212             dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
 213     selection.add_option('--playlist-end',
 214             dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
 215     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 216     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 217     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
 218     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 219     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 220     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 221     selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
 222     selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 223     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 224     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 225                          help='download only videos suitable for the given age',
 226                          default=None, type=int)
 227     selection.add_option('--download-archive', metavar='FILE',
 228                          dest='download_archive',
 229                          help='Download only videos not present in the archive file. Record all downloaded videos in it.')
 230
 231
 232     authentication.add_option('-u', '--username',
 233             dest='username', metavar='USERNAME', help='account username')
 234     authentication.add_option('-p', '--password',
 235             dest='password', metavar='PASSWORD', help='account password')
 236     authentication.add_option('-n', '--netrc',
 237             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 238     authentication.add_option('--video-password',
 239             dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)')
 240
 241
 242     video_format.add_option('-f', '--format',
 243             action='store', dest='format', metavar='FORMAT', default='best',
 244             help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
 245     video_format.add_option('--all-formats',
 246             action='store_const', dest='format', help='download all available video formats', const='all')
 247     video_format.add_option('--prefer-free-formats',
 248             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 249     video_format.add_option('--max-quality',
 250             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 251     video_format.add_option('-F', '--list-formats',
 252             action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 253
 254     subtitles.add_option('--write-sub', '--write-srt',
 255             action='store_true', dest='writesubtitles',
 256             help='write subtitle file', default=False)
 257     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 258             action='store_true', dest='writeautomaticsub',
 259             help='write automatic subtitle file (youtube only)', default=False)
 260     subtitles.add_option('--all-subs',
 261             action='store_true', dest='allsubtitles',
 262             help='downloads all the available subtitles of the video', default=False)
 263     subtitles.add_option('--list-subs',
 264             action='store_true', dest='listsubtitles',
 265             help='lists all available subtitles for the video', default=False)
 266     subtitles.add_option('--sub-format',
 267             action='store', dest='subtitlesformat', metavar='FORMAT',
 268             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 269     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 270             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 271             default=[], callback=_comma_separated_values_options_callback,
 272             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 273
 274     downloader.add_option('-r', '--rate-limit',
 275             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 276     downloader.add_option('-R', '--retries',
 277             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 278     downloader.add_option('--buffer-size',
 279             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 280     downloader.add_option('--no-resize-buffer',
 281             action='store_true', dest='noresizebuffer',
 282             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 283     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 284
 285     verbosity.add_option('-q', '--quiet',
 286             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 287     verbosity.add_option('-s', '--simulate',
 288             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 289     verbosity.add_option('--skip-download',
 290             action='store_true', dest='skip_download', help='do not download the video', default=False)
 291     verbosity.add_option('-g', '--get-url',
 292             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 293     verbosity.add_option('-e', '--get-title',
 294             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 295     verbosity.add_option('--get-id',
 296             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 297     verbosity.add_option('--get-thumbnail',
 298             action='store_true', dest='getthumbnail',
 299             help='simulate, quiet but print thumbnail URL', default=False)
 300     verbosity.add_option('--get-description',
 301             action='store_true', dest='getdescription',
 302             help='simulate, quiet but print video description', default=False)
 303     verbosity.add_option('--get-filename',
 304             action='store_true', dest='getfilename',
 305             help='simulate, quiet but print output filename', default=False)
 306     verbosity.add_option('--get-format',
 307             action='store_true', dest='getformat',
 308             help='simulate, quiet but print output format', default=False)
 309     verbosity.add_option('--newline',
 310             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 311     verbosity.add_option('--no-progress',
 312             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 313     verbosity.add_option('--console-title',
 314             action='store_true', dest='consoletitle',
 315             help='display progress in console titlebar', default=False)
 316     verbosity.add_option('-v', '--verbose',
 317             action='store_true', dest='verbose', help='print various debugging information', default=False)
 318     verbosity.add_option('--dump-intermediate-pages',
 319             action='store_true', dest='dump_intermediate_pages', default=False,
 320             help='print downloaded pages to debug problems(very verbose)')
 321     verbosity.add_option('--write-pages',
 322             action='store_true', dest='write_pages', default=False,
 323             help='Write downloaded pages to files in the current directory')
 324     verbosity.add_option('--youtube-print-sig-code',
 325             action='store_true', dest='youtube_print_sig_code', default=False,
 326             help=optparse.SUPPRESS_HELP)
 327
 328
 329     filesystem.add_option('-t', '--title',
 330             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 331     filesystem.add_option('--id',
 332             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 333     filesystem.add_option('-l', '--literal',
 334             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 335     filesystem.add_option('-A', '--auto-number',
 336             action='store_true', dest='autonumber',
 337             help='number downloaded files starting from 00000', default=False)
 338     filesystem.add_option('-o', '--output',
 339             dest='outtmpl', metavar='TEMPLATE',
 340             help=('output filename template. Use %(title)s to get the title, '
 341                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 342                   '%(autonumber)s to get an automatically incremented number, '
 343                   '%(ext)s for the filename extension, '
 344                   '%(format)s for the format description (like "22 - 1280x720" or "HD"),'
 345                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"),'
 346                   '%(upload_date)s for the upload date (YYYYMMDD), '
 347                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 348                   '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
 349                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 350                   'Use - to output to stdout. Can also be used to download to a different directory, '
 351                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 352     filesystem.add_option('--autonumber-size',
 353             dest='autonumber_size', metavar='NUMBER',
 354             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 355     filesystem.add_option('--restrict-filenames',
 356             action='store_true', dest='restrictfilenames',
 357             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 358     filesystem.add_option('-a', '--batch-file',
 359             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 360     filesystem.add_option('-w', '--no-overwrites',
 361             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 362     filesystem.add_option('-c', '--continue',
 363             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 364     filesystem.add_option('--no-continue',
 365             action='store_false', dest='continue_dl',
 366             help='do not resume partially downloaded files (restart from beginning)')
 367     filesystem.add_option('--cookies',
 368             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 369     filesystem.add_option('--no-part',
 370             action='store_true', dest='nopart', help='do not use .part files', default=False)
 371     filesystem.add_option('--no-mtime',
 372             action='store_false', dest='updatetime',
 373             help='do not use the Last-modified header to set the file modification time', default=True)
 374     filesystem.add_option('--write-description',
 375             action='store_true', dest='writedescription',
 376             help='write video description to a .description file', default=False)
 377     filesystem.add_option('--write-info-json',
 378             action='store_true', dest='writeinfojson',
 379             help='write video metadata to a .info.json file', default=False)
 380     filesystem.add_option('--write-annotations',
 381             action='store_true', dest='writeannotations',
 382             help='write video annotations to a .annotation file', default=False)
 383     filesystem.add_option('--write-thumbnail',
 384             action='store_true', dest='writethumbnail',
 385             help='write thumbnail image to disk', default=False)
 386
 387
 388     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 389             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 390     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 391             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 392     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 393             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 394     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 395             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 396     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 397             help='keeps the video file on disk after the post-processing; the video is erased by default')
 398     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 399             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 400     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 401             help='embed subtitles in the video (only for mp4 videos)')
 402     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 403             help='add metadata to the files')
 404
 405
 406     parser.add_option_group(general)
 407     parser.add_option_group(selection)
 408     parser.add_option_group(downloader)
 409     parser.add_option_group(filesystem)
 410     parser.add_option_group(verbosity)
 411     parser.add_option_group(video_format)
 412     parser.add_option_group(subtitles)
 413     parser.add_option_group(authentication)
 414     parser.add_option_group(postproc)
 415
 416     if overrideArguments is not None:
 417         opts, args = parser.parse_args(overrideArguments)
 418         if opts.verbose:
 419             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 420     else:
 421         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 422         if xdg_config_home:
 423             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 424             if not os.path.isfile(userConfFile):
 425                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 426         else:
 427             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 428             if not os.path.isfile(userConfFile):
 429                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 430         systemConf = _readOptions('/etc/youtube-dl.conf')
 431         userConf = _readOptions(userConfFile)
 432         commandLineConf = sys.argv[1:]
 433         argv = systemConf + userConf + commandLineConf
 434         opts, args = parser.parse_args(argv)
 435         if opts.verbose:
 436             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 437             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 438             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 439
 440     return parser, opts, args
 441
 442 def _real_main(argv=None):
 443     # Compatibility fixes for Windows
 444     if sys.platform == 'win32':
 445         # https://github.com/rg3/youtube-dl/issues/820
 446         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 447
 448     parser, opts, args = parseOpts(argv)
 449
 450     # Open appropriate CookieJar
 451     if opts.cookiefile is None:
 452         jar = compat_cookiejar.CookieJar()
 453     else:
 454         try:
 455             jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
 456             if os.access(opts.cookiefile, os.R_OK):
 457                 jar.load()
 458         except (IOError, OSError) as err:
 459             if opts.verbose:
 460                 traceback.print_exc()
 461             write_string(u'ERROR: unable to open cookie file\n')
 462             sys.exit(101)
 463     # Set user agent
 464     if opts.user_agent is not None:
 465         std_headers['User-Agent'] = opts.user_agent
 466
 467     # Set referer
 468     if opts.referer is not None:
 469         std_headers['Referer'] = opts.referer
 470
 471     # Dump user agent
 472     if opts.dump_user_agent:
 473         compat_print(std_headers['User-Agent'])
 474         sys.exit(0)
 475
 476     # Batch file verification
 477     batchurls = []
 478     if opts.batchfile is not None:
 479         try:
 480             if opts.batchfile == '-':
 481                 batchfd = sys.stdin
 482             else:
 483                 batchfd = open(opts.batchfile, 'r')
 484             batchurls = batchfd.readlines()
 485             batchurls = [x.strip() for x in batchurls]
 486             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 487             if opts.verbose:
 488                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 489         except IOError:
 490             sys.exit(u'ERROR: batch file could not be read')
 491     all_urls = batchurls + args
 492     all_urls = [url.strip() for url in all_urls]
 493
 494     opener = _setup_opener(jar=jar, opts=opts)
 495
 496     extractors = gen_extractors()
 497
 498     if opts.list_extractors:
 499         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 500             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 501             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 502             all_urls = [url for url in all_urls if url not in matchedUrls]
 503             for mu in matchedUrls:
 504                 compat_print(u'  ' + mu)
 505         sys.exit(0)
 506     if opts.list_extractor_descriptions:
 507         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 508             if not ie._WORKING:
 509                 continue
 510             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 511             if desc is False:
 512                 continue
 513             if hasattr(ie, 'SEARCH_KEY'):
 514                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 515                 _COUNTS = (u'', u'5', u'10', u'all')
 516                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 517             compat_print(desc)
 518         sys.exit(0)
 519
 520
 521     # Conflicting, missing and erroneous options
 522     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 523         parser.error(u'using .netrc conflicts with giving username/password')
 524     if opts.password is not None and opts.username is None:
 525         parser.error(u' account username missing\n')
 526     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 527         parser.error(u'using output template conflicts with using title, video ID or auto number')
 528     if opts.usetitle and opts.useid:
 529         parser.error(u'using title conflicts with using video ID')
 530     if opts.username is not None and opts.password is None:
 531         opts.password = getpass.getpass(u'Type account password and press return:')
 532     if opts.ratelimit is not None:
 533         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 534         if numeric_limit is None:
 535             parser.error(u'invalid rate limit specified')
 536         opts.ratelimit = numeric_limit
 537     if opts.min_filesize is not None:
 538         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 539         if numeric_limit is None:
 540             parser.error(u'invalid min_filesize specified')
 541         opts.min_filesize = numeric_limit
 542     if opts.max_filesize is not None:
 543         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 544         if numeric_limit is None:
 545             parser.error(u'invalid max_filesize specified')
 546         opts.max_filesize = numeric_limit
 547     if opts.retries is not None:
 548         try:
 549             opts.retries = int(opts.retries)
 550         except (TypeError, ValueError) as err:
 551             parser.error(u'invalid retry count specified')
 552     if opts.buffersize is not None:
 553         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 554         if numeric_buffersize is None:
 555             parser.error(u'invalid buffer size specified')
 556         opts.buffersize = numeric_buffersize
 557     try:
 558         opts.playliststart = int(opts.playliststart)
 559         if opts.playliststart <= 0:
 560             raise ValueError(u'Playlist start must be positive')
 561     except (TypeError, ValueError) as err:
 562         parser.error(u'invalid playlist start number specified')
 563     try:
 564         opts.playlistend = int(opts.playlistend)
 565         if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
 566             raise ValueError(u'Playlist end must be greater than playlist start')
 567     except (TypeError, ValueError) as err:
 568         parser.error(u'invalid playlist end number specified')
 569     if opts.extractaudio:
 570         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 571             parser.error(u'invalid audio format specified')
 572     if opts.audioquality:
 573         opts.audioquality = opts.audioquality.strip('k').strip('K')
 574         if not opts.audioquality.isdigit():
 575             parser.error(u'invalid audio quality specified')
 576     if opts.recodevideo is not None:
 577         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 578             parser.error(u'invalid video recode format specified')
 579     if opts.date is not None:
 580         date = DateRange.day(opts.date)
 581     else:
 582         date = DateRange(opts.dateafter, opts.datebefore)
 583
 584     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 585     # this was the old behaviour if only --all-sub was given.
 586     if opts.allsubtitles and (opts.writeautomaticsub == False):
 587         opts.writesubtitles = True
 588
 589     if sys.version_info < (3,):
 590         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 591         if opts.outtmpl is not None:
 592             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 593     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 594             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 595             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 596             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 597             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 598             or (opts.useid and u'%(id)s.%(ext)s')
 599             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 600             or u'%(title)s-%(id)s.%(ext)s')
 601     if '%(ext)s' not in outtmpl and opts.extractaudio:
 602         parser.error(u'Cannot download a video and extract audio into the same'
 603                      u' file! Use "%%(ext)s" instead of %r' %
 604                      determine_ext(outtmpl, u''))
 605
 606     ydl_opts = {
 607         'usenetrc': opts.usenetrc,
 608         'username': opts.username,
 609         'password': opts.password,
 610         'videopassword': opts.videopassword,
 611         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 612         'forceurl': opts.geturl,
 613         'forcetitle': opts.gettitle,
 614         'forceid': opts.getid,
 615         'forcethumbnail': opts.getthumbnail,
 616         'forcedescription': opts.getdescription,
 617         'forcefilename': opts.getfilename,
 618         'forceformat': opts.getformat,
 619         'simulate': opts.simulate,
 620         'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 621         'format': opts.format,
 622         'format_limit': opts.format_limit,
 623         'listformats': opts.listformats,
 624         'outtmpl': outtmpl,
 625         'autonumber_size': opts.autonumber_size,
 626         'restrictfilenames': opts.restrictfilenames,
 627         'ignoreerrors': opts.ignoreerrors,
 628         'ratelimit': opts.ratelimit,
 629         'nooverwrites': opts.nooverwrites,
 630         'retries': opts.retries,
 631         'buffersize': opts.buffersize,
 632         'noresizebuffer': opts.noresizebuffer,
 633         'continuedl': opts.continue_dl,
 634         'noprogress': opts.noprogress,
 635         'progress_with_newline': opts.progress_with_newline,
 636         'playliststart': opts.playliststart,
 637         'playlistend': opts.playlistend,
 638         'noplaylist': opts.noplaylist,
 639         'logtostderr': opts.outtmpl == '-',
 640         'consoletitle': opts.consoletitle,
 641         'nopart': opts.nopart,
 642         'updatetime': opts.updatetime,
 643         'writedescription': opts.writedescription,
 644         'writeannotations': opts.writeannotations,
 645         'writeinfojson': opts.writeinfojson,
 646         'writethumbnail': opts.writethumbnail,
 647         'writesubtitles': opts.writesubtitles,
 648         'writeautomaticsub': opts.writeautomaticsub,
 649         'allsubtitles': opts.allsubtitles,
 650         'listsubtitles': opts.listsubtitles,
 651         'subtitlesformat': opts.subtitlesformat,
 652         'subtitleslangs': opts.subtitleslangs,
 653         'matchtitle': decodeOption(opts.matchtitle),
 654         'rejecttitle': decodeOption(opts.rejecttitle),
 655         'max_downloads': opts.max_downloads,
 656         'prefer_free_formats': opts.prefer_free_formats,
 657         'verbose': opts.verbose,
 658         'dump_intermediate_pages': opts.dump_intermediate_pages,
 659         'write_pages': opts.write_pages,
 660         'test': opts.test,
 661         'keepvideo': opts.keepvideo,
 662         'min_filesize': opts.min_filesize,
 663         'max_filesize': opts.max_filesize,
 664         'daterange': date,
 665         'cachedir': opts.cachedir,
 666         'youtube_print_sig_code': opts.youtube_print_sig_code,
 667         'age_limit': opts.age_limit,
 668         'download_archive': opts.download_archive,
 669     }
 670
 671     with YoutubeDL(ydl_opts) as ydl:
 672         if opts.verbose:
 673             write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
 674             try:
 675                 sp = subprocess.Popen(
 676                     ['git', 'rev-parse', '--short', 'HEAD'],
 677                     stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 678                     cwd=os.path.dirname(os.path.abspath(__file__)))
 679                 out, err = sp.communicate()
 680                 out = out.decode().strip()
 681                 if re.match('[0-9a-f]+', out):
 682                     write_string(u'[debug] Git HEAD: ' + out + u'\n')
 683             except:
 684                 try:
 685                     sys.exc_clear()
 686                 except:
 687                     pass
 688             write_string(u'[debug] Python version %s - %s' %
 689                          (platform.python_version(), platform_name()) + u'\n')
 690
 691             proxy_map = {}
 692             for handler in opener.handlers:
 693                 if hasattr(handler, 'proxies'):
 694                     proxy_map.update(handler.proxies)
 695             write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
 696
 697         ydl.add_default_info_extractors()
 698
 699         # PostProcessors
 700         # Add the metadata pp first, the other pps will copy it
 701         if opts.addmetadata:
 702             ydl.add_post_processor(FFmpegMetadataPP())
 703         if opts.extractaudio:
 704             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 705         if opts.recodevideo:
 706             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 707         if opts.embedsubtitles:
 708             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 709
 710         # Update version
 711         if opts.update_self:
 712             update_self(ydl.to_screen, opts.verbose)
 713
 714         # Maybe do nothing
 715         if len(all_urls) < 1:
 716             if not opts.update_self:
 717                 parser.error(u'you must provide at least one URL')
 718             else:
 719                 sys.exit()
 720
 721         try:
 722             retcode = ydl.download(all_urls)
 723         except MaxDownloadsReached:
 724             ydl.to_screen(u'--max-download limit reached, aborting.')
 725             retcode = 101
 726
 727     # Dump cookie jar if requested
 728     if opts.cookiefile is not None:
 729         try:
 730             jar.save()
 731         except (IOError, OSError):
 732             sys.exit(u'ERROR: unable to save cookie jar')
 733
 734     sys.exit(retcode)
 735
 736
 737 def _setup_opener(jar=None, opts=None, timeout=300):
 738     if opts is None:
 739         FakeOptions = collections.namedtuple(
 740             'FakeOptions', ['proxy', 'no_check_certificate'])
 741         opts = FakeOptions(proxy=None, no_check_certificate=False)
 742
 743     cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
 744     if opts.proxy is not None:
 745         if opts.proxy == '':
 746             proxies = {}
 747         else:
 748             proxies = {'http': opts.proxy, 'https': opts.proxy}
 749     else:
 750         proxies = compat_urllib_request.getproxies()
 751         # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
 752         if 'http' in proxies and 'https' not in proxies:
 753             proxies['https'] = proxies['http']
 754     proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 755     https_handler = make_HTTPS_handler(opts)
 756     opener = compat_urllib_request.build_opener(
 757         https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
 758     # Delete the default user-agent header, which would otherwise apply in
 759     # cases where our custom HTTP handler doesn't come into play
 760     # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
 761     opener.addheaders = []
 762     compat_urllib_request.install_opener(opener)
 763     socket.setdefaulttimeout(timeout)
 764     return opener
 765
 766
 767 def main(argv=None):
 768     try:
 769         _real_main(argv)
 770     except DownloadError:
 771         sys.exit(1)
 772     except SameFileError:
 773         sys.exit(u'ERROR: fixed output name but more than one file to download')
 774     except KeyboardInterrupt:
 775         sys.exit(u'\nERROR: Interrupted by user')