youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34 )
  35
  36 __license__ = 'Public Domain'
  37
  38 import codecs
  39 import collections
  40 import getpass
  41 import optparse
  42 import os
  43 import random
  44 import re
  45 import shlex
  46 import socket
  47 import subprocess
  48 import sys
  49 import warnings
  50 import platform
  51
  52
  53 from .utils import *
  54 from .update import update_self
  55 from .version import __version__
  56 from .FileDownloader import *
  57 from .extractor import gen_extractors
  58 from .YoutubeDL import YoutubeDL
  59 from .PostProcessor import *
  60
  61 def parseOpts(overrideArguments=None):
  62     def _readOptions(filename_bytes):
  63         try:
  64             optionf = open(filename_bytes)
  65         except IOError:
  66             return [] # silently skip if file is not present
  67         try:
  68             res = []
  69             for l in optionf:
  70                 res += shlex.split(l, comments=True)
  71         finally:
  72             optionf.close()
  73         return res
  74
  75     def _format_option_string(option):
  76         ''' ('-o', '--option') -> -o, --format METAVAR'''
  77
  78         opts = []
  79
  80         if option._short_opts:
  81             opts.append(option._short_opts[0])
  82         if option._long_opts:
  83             opts.append(option._long_opts[0])
  84         if len(opts) > 1:
  85             opts.insert(1, ', ')
  86
  87         if option.takes_value(): opts.append(' %s' % option.metavar)
  88
  89         return "".join(opts)
  90
  91     def _comma_separated_values_options_callback(option, opt_str, value, parser):
  92         setattr(parser.values, option.dest, value.split(','))
  93
  94     def _find_term_columns():
  95         columns = os.environ.get('COLUMNS', None)
  96         if columns:
  97             return int(columns)
  98
  99         try:
 100             sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 101             out,err = sp.communicate()
 102             return int(out.split()[1])
 103         except:
 104             pass
 105         return None
 106
 107     def _hide_login_info(opts):
 108         opts = list(opts)
 109         for private_opt in ['-p', '--password', '-u', '--username']:
 110             try:
 111                 i = opts.index(private_opt)
 112                 opts[i+1] = '<PRIVATE>'
 113             except ValueError:
 114                 pass
 115         return opts
 116
 117     max_width = 80
 118     max_help_position = 80
 119
 120     # No need to wrap help messages if we're on a wide console
 121     columns = _find_term_columns()
 122     if columns: max_width = columns
 123
 124     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 125     fmt.format_option_strings = _format_option_string
 126
 127     kw = {
 128         'version'   : __version__,
 129         'formatter' : fmt,
 130         'usage' : '%prog [options] url [url...]',
 131         'conflict_handler' : 'resolve',
 132     }
 133
 134     parser = optparse.OptionParser(**kw)
 135
 136     # option groups
 137     general        = optparse.OptionGroup(parser, 'General Options')
 138     selection      = optparse.OptionGroup(parser, 'Video Selection')
 139     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 140     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 141     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 142     downloader     = optparse.OptionGroup(parser, 'Download Options')
 143     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 144     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 145     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 146
 147     general.add_option('-h', '--help',
 148             action='help', help='print this help text and exit')
 149     general.add_option('-v', '--version',
 150             action='version', help='print program version and exit')
 151     general.add_option('-U', '--update',
 152             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 153     general.add_option('-i', '--ignore-errors',
 154             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 155     general.add_option('--dump-user-agent',
 156             action='store_true', dest='dump_user_agent',
 157             help='display the current browser identification', default=False)
 158     general.add_option('--user-agent',
 159             dest='user_agent', help='specify a custom user agent', metavar='UA')
 160     general.add_option('--referer',
 161             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 162             metavar='REF', default=None)
 163     general.add_option('--list-extractors',
 164             action='store_true', dest='list_extractors',
 165             help='List all supported extractors and the URLs they would handle', default=False)
 166     general.add_option('--extractor-descriptions',
 167             action='store_true', dest='list_extractor_descriptions',
 168             help='Output descriptions of all supported extractors', default=False)
 169     general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
 170     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 171     general.add_option(
 172         '--cache-dir', dest='cachedir', default=get_cachedir(),
 173         help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
 174     general.add_option(
 175         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 176         help='Disable filesystem caching')
 177
 178
 179     selection.add_option('--playlist-start',
 180             dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
 181     selection.add_option('--playlist-end',
 182             dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
 183     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 184     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 185     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
 186     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 187     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 188     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 189     selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
 190     selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 191     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 192     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 193                          help='download only videos suitable for the given age',
 194                          default=None, type=int)
 195     selection.add_option('--download-archive', metavar='FILE',
 196                          dest='download_archive',
 197                          help='Download only videos not present in the archive file. Record all downloaded videos in it.')
 198
 199
 200     authentication.add_option('-u', '--username',
 201             dest='username', metavar='USERNAME', help='account username')
 202     authentication.add_option('-p', '--password',
 203             dest='password', metavar='PASSWORD', help='account password')
 204     authentication.add_option('-n', '--netrc',
 205             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 206     authentication.add_option('--video-password',
 207             dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)')
 208
 209
 210     video_format.add_option('-f', '--format',
 211             action='store', dest='format', metavar='FORMAT',
 212             help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
 213     video_format.add_option('--all-formats',
 214             action='store_const', dest='format', help='download all available video formats', const='all')
 215     video_format.add_option('--prefer-free-formats',
 216             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 217     video_format.add_option('--max-quality',
 218             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 219     video_format.add_option('-F', '--list-formats',
 220             action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 221
 222     subtitles.add_option('--write-sub', '--write-srt',
 223             action='store_true', dest='writesubtitles',
 224             help='write subtitle file', default=False)
 225     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 226             action='store_true', dest='writeautomaticsub',
 227             help='write automatic subtitle file (youtube only)', default=False)
 228     subtitles.add_option('--all-subs',
 229             action='store_true', dest='allsubtitles',
 230             help='downloads all the available subtitles of the video', default=False)
 231     subtitles.add_option('--list-subs',
 232             action='store_true', dest='listsubtitles',
 233             help='lists all available subtitles for the video', default=False)
 234     subtitles.add_option('--sub-format',
 235             action='store', dest='subtitlesformat', metavar='FORMAT',
 236             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 237     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 238             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 239             default=[], callback=_comma_separated_values_options_callback,
 240             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 241
 242     downloader.add_option('-r', '--rate-limit',
 243             dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)')
 244     downloader.add_option('-R', '--retries',
 245             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 246     downloader.add_option('--buffer-size',
 247             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024")
 248     downloader.add_option('--no-resize-buffer',
 249             action='store_true', dest='noresizebuffer',
 250             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 251     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 252
 253     verbosity.add_option('-q', '--quiet',
 254             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 255     verbosity.add_option('-s', '--simulate',
 256             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 257     verbosity.add_option('--skip-download',
 258             action='store_true', dest='skip_download', help='do not download the video', default=False)
 259     verbosity.add_option('-g', '--get-url',
 260             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 261     verbosity.add_option('-e', '--get-title',
 262             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 263     verbosity.add_option('--get-id',
 264             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 265     verbosity.add_option('--get-thumbnail',
 266             action='store_true', dest='getthumbnail',
 267             help='simulate, quiet but print thumbnail URL', default=False)
 268     verbosity.add_option('--get-description',
 269             action='store_true', dest='getdescription',
 270             help='simulate, quiet but print video description', default=False)
 271     verbosity.add_option('--get-filename',
 272             action='store_true', dest='getfilename',
 273             help='simulate, quiet but print output filename', default=False)
 274     verbosity.add_option('--get-format',
 275             action='store_true', dest='getformat',
 276             help='simulate, quiet but print output format', default=False)
 277     verbosity.add_option('--newline',
 278             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 279     verbosity.add_option('--no-progress',
 280             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 281     verbosity.add_option('--console-title',
 282             action='store_true', dest='consoletitle',
 283             help='display progress in console titlebar', default=False)
 284     verbosity.add_option('-v', '--verbose',
 285             action='store_true', dest='verbose', help='print various debugging information', default=False)
 286     verbosity.add_option('--dump-intermediate-pages',
 287             action='store_true', dest='dump_intermediate_pages', default=False,
 288             help='print downloaded pages to debug problems(very verbose)')
 289     verbosity.add_option('--youtube-print-sig-code',
 290             action='store_true', dest='youtube_print_sig_code', default=False,
 291             help=optparse.SUPPRESS_HELP)
 292
 293
 294     filesystem.add_option('-t', '--title',
 295             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 296     filesystem.add_option('--id',
 297             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 298     filesystem.add_option('-l', '--literal',
 299             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 300     filesystem.add_option('-A', '--auto-number',
 301             action='store_true', dest='autonumber',
 302             help='number downloaded files starting from 00000', default=False)
 303     filesystem.add_option('-o', '--output',
 304             dest='outtmpl', metavar='TEMPLATE',
 305             help=('output filename template. Use %(title)s to get the title, '
 306                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 307                   '%(autonumber)s to get an automatically incremented number, '
 308                   '%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), '
 309                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 310                   '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
 311                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 312                   'Use - to output to stdout. Can also be used to download to a different directory, '
 313                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 314     filesystem.add_option('--autonumber-size',
 315             dest='autonumber_size', metavar='NUMBER',
 316             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given')
 317     filesystem.add_option('--restrict-filenames',
 318             action='store_true', dest='restrictfilenames',
 319             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 320     filesystem.add_option('-a', '--batch-file',
 321             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 322     filesystem.add_option('-w', '--no-overwrites',
 323             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 324     filesystem.add_option('-c', '--continue',
 325             action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
 326     filesystem.add_option('--no-continue',
 327             action='store_false', dest='continue_dl',
 328             help='do not resume partially downloaded files (restart from beginning)')
 329     filesystem.add_option('--cookies',
 330             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 331     filesystem.add_option('--no-part',
 332             action='store_true', dest='nopart', help='do not use .part files', default=False)
 333     filesystem.add_option('--no-mtime',
 334             action='store_false', dest='updatetime',
 335             help='do not use the Last-modified header to set the file modification time', default=True)
 336     filesystem.add_option('--write-description',
 337             action='store_true', dest='writedescription',
 338             help='write video description to a .description file', default=False)
 339     filesystem.add_option('--write-info-json',
 340             action='store_true', dest='writeinfojson',
 341             help='write video metadata to a .info.json file', default=False)
 342     filesystem.add_option('--write-thumbnail',
 343             action='store_true', dest='writethumbnail',
 344             help='write thumbnail image to disk', default=False)
 345
 346
 347     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 348             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 349     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 350             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 351     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 352             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 353     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 354             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 355     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 356             help='keeps the video file on disk after the post-processing; the video is erased by default')
 357     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 358             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 359     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 360             help='embed subtitles in the video (only for mp4 videos)')
 361
 362
 363     parser.add_option_group(general)
 364     parser.add_option_group(selection)
 365     parser.add_option_group(downloader)
 366     parser.add_option_group(filesystem)
 367     parser.add_option_group(verbosity)
 368     parser.add_option_group(video_format)
 369     parser.add_option_group(subtitles)
 370     parser.add_option_group(authentication)
 371     parser.add_option_group(postproc)
 372
 373     if overrideArguments is not None:
 374         opts, args = parser.parse_args(overrideArguments)
 375         if opts.verbose:
 376             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 377     else:
 378         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 379         if xdg_config_home:
 380             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 381             if not os.path.isfile(userConfFile):
 382                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 383         else:
 384             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 385             if not os.path.isfile(userConfFile):
 386                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 387         systemConf = _readOptions('/etc/youtube-dl.conf')
 388         userConf = _readOptions(userConfFile)
 389         commandLineConf = sys.argv[1:]
 390         argv = systemConf + userConf + commandLineConf
 391         opts, args = parser.parse_args(argv)
 392         if opts.verbose:
 393             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 394             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 395             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 396
 397     return parser, opts, args
 398
 399 def _real_main(argv=None):
 400     # Compatibility fixes for Windows
 401     if sys.platform == 'win32':
 402         # https://github.com/rg3/youtube-dl/issues/820
 403         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 404
 405     parser, opts, args = parseOpts(argv)
 406
 407     # Open appropriate CookieJar
 408     if opts.cookiefile is None:
 409         jar = compat_cookiejar.CookieJar()
 410     else:
 411         try:
 412             jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
 413             if os.access(opts.cookiefile, os.R_OK):
 414                 jar.load()
 415         except (IOError, OSError) as err:
 416             if opts.verbose:
 417                 traceback.print_exc()
 418             write_string(u'ERROR: unable to open cookie file\n')
 419             sys.exit(101)
 420     # Set user agent
 421     if opts.user_agent is not None:
 422         std_headers['User-Agent'] = opts.user_agent
 423
 424     # Set referer
 425     if opts.referer is not None:
 426         std_headers['Referer'] = opts.referer
 427
 428     # Dump user agent
 429     if opts.dump_user_agent:
 430         compat_print(std_headers['User-Agent'])
 431         sys.exit(0)
 432
 433     # Batch file verification
 434     batchurls = []
 435     if opts.batchfile is not None:
 436         try:
 437             if opts.batchfile == '-':
 438                 batchfd = sys.stdin
 439             else:
 440                 batchfd = open(opts.batchfile, 'r')
 441             batchurls = batchfd.readlines()
 442             batchurls = [x.strip() for x in batchurls]
 443             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 444             if opts.verbose:
 445                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 446         except IOError:
 447             sys.exit(u'ERROR: batch file could not be read')
 448     all_urls = batchurls + args
 449     all_urls = [url.strip() for url in all_urls]
 450
 451     opener = _setup_opener(jar=jar, opts=opts)
 452
 453     extractors = gen_extractors()
 454
 455     if opts.list_extractors:
 456         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 457             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 458             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 459             all_urls = [url for url in all_urls if url not in matchedUrls]
 460             for mu in matchedUrls:
 461                 compat_print(u'  ' + mu)
 462         sys.exit(0)
 463     if opts.list_extractor_descriptions:
 464         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 465             if not ie._WORKING:
 466                 continue
 467             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 468             if desc is False:
 469                 continue
 470             if hasattr(ie, 'SEARCH_KEY'):
 471                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 472                 _COUNTS = (u'', u'5', u'10', u'all')
 473                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 474             compat_print(desc)
 475         sys.exit(0)
 476
 477
 478     # Conflicting, missing and erroneous options
 479     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 480         parser.error(u'using .netrc conflicts with giving username/password')
 481     if opts.password is not None and opts.username is None:
 482         parser.error(u' account username missing\n')
 483     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 484         parser.error(u'using output template conflicts with using title, video ID or auto number')
 485     if opts.usetitle and opts.useid:
 486         parser.error(u'using title conflicts with using video ID')
 487     if opts.username is not None and opts.password is None:
 488         opts.password = getpass.getpass(u'Type account password and press return:')
 489     if opts.ratelimit is not None:
 490         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 491         if numeric_limit is None:
 492             parser.error(u'invalid rate limit specified')
 493         opts.ratelimit = numeric_limit
 494     if opts.min_filesize is not None:
 495         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 496         if numeric_limit is None:
 497             parser.error(u'invalid min_filesize specified')
 498         opts.min_filesize = numeric_limit
 499     if opts.max_filesize is not None:
 500         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 501         if numeric_limit is None:
 502             parser.error(u'invalid max_filesize specified')
 503         opts.max_filesize = numeric_limit
 504     if opts.retries is not None:
 505         try:
 506             opts.retries = int(opts.retries)
 507         except (TypeError, ValueError) as err:
 508             parser.error(u'invalid retry count specified')
 509     if opts.buffersize is not None:
 510         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 511         if numeric_buffersize is None:
 512             parser.error(u'invalid buffer size specified')
 513         opts.buffersize = numeric_buffersize
 514     try:
 515         opts.playliststart = int(opts.playliststart)
 516         if opts.playliststart <= 0:
 517             raise ValueError(u'Playlist start must be positive')
 518     except (TypeError, ValueError) as err:
 519         parser.error(u'invalid playlist start number specified')
 520     try:
 521         opts.playlistend = int(opts.playlistend)
 522         if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
 523             raise ValueError(u'Playlist end must be greater than playlist start')
 524     except (TypeError, ValueError) as err:
 525         parser.error(u'invalid playlist end number specified')
 526     if opts.extractaudio:
 527         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 528             parser.error(u'invalid audio format specified')
 529     if opts.audioquality:
 530         opts.audioquality = opts.audioquality.strip('k').strip('K')
 531         if not opts.audioquality.isdigit():
 532             parser.error(u'invalid audio quality specified')
 533     if opts.recodevideo is not None:
 534         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 535             parser.error(u'invalid video recode format specified')
 536     if opts.date is not None:
 537         date = DateRange.day(opts.date)
 538     else:
 539         date = DateRange(opts.dateafter, opts.datebefore)
 540
 541     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 542     # this was the old behaviour if only --all-sub was given.
 543     if opts.allsubtitles and (opts.writeautomaticsub == False):
 544         opts.writesubtitles = True
 545
 546     if sys.version_info < (3,):
 547         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 548         if opts.outtmpl is not None:
 549             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 550     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 551             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 552             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 553             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 554             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 555             or (opts.useid and u'%(id)s.%(ext)s')
 556             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 557             or u'%(title)s-%(id)s.%(ext)s')
 558     if '%(ext)s' not in outtmpl and opts.extractaudio:
 559         parser.error(u'Cannot download a video and extract audio into the same'
 560                      u' file! Use "%%(ext)s" instead of %r' %
 561                      determine_ext(outtmpl, u''))
 562
 563     # YoutubeDL
 564     ydl = YoutubeDL({
 565         'usenetrc': opts.usenetrc,
 566         'username': opts.username,
 567         'password': opts.password,
 568         'videopassword': opts.videopassword,
 569         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 570         'forceurl': opts.geturl,
 571         'forcetitle': opts.gettitle,
 572         'forceid': opts.getid,
 573         'forcethumbnail': opts.getthumbnail,
 574         'forcedescription': opts.getdescription,
 575         'forcefilename': opts.getfilename,
 576         'forceformat': opts.getformat,
 577         'simulate': opts.simulate,
 578         'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 579         'format': opts.format,
 580         'format_limit': opts.format_limit,
 581         'listformats': opts.listformats,
 582         'outtmpl': outtmpl,
 583         'autonumber_size': opts.autonumber_size,
 584         'restrictfilenames': opts.restrictfilenames,
 585         'ignoreerrors': opts.ignoreerrors,
 586         'ratelimit': opts.ratelimit,
 587         'nooverwrites': opts.nooverwrites,
 588         'retries': opts.retries,
 589         'buffersize': opts.buffersize,
 590         'noresizebuffer': opts.noresizebuffer,
 591         'continuedl': opts.continue_dl,
 592         'noprogress': opts.noprogress,
 593         'progress_with_newline': opts.progress_with_newline,
 594         'playliststart': opts.playliststart,
 595         'playlistend': opts.playlistend,
 596         'noplaylist': opts.noplaylist,
 597         'logtostderr': opts.outtmpl == '-',
 598         'consoletitle': opts.consoletitle,
 599         'nopart': opts.nopart,
 600         'updatetime': opts.updatetime,
 601         'writedescription': opts.writedescription,
 602         'writeinfojson': opts.writeinfojson,
 603         'writethumbnail': opts.writethumbnail,
 604         'writesubtitles': opts.writesubtitles,
 605         'writeautomaticsub': opts.writeautomaticsub,
 606         'allsubtitles': opts.allsubtitles,
 607         'listsubtitles': opts.listsubtitles,
 608         'subtitlesformat': opts.subtitlesformat,
 609         'subtitleslangs': opts.subtitleslangs,
 610         'matchtitle': decodeOption(opts.matchtitle),
 611         'rejecttitle': decodeOption(opts.rejecttitle),
 612         'max_downloads': opts.max_downloads,
 613         'prefer_free_formats': opts.prefer_free_formats,
 614         'verbose': opts.verbose,
 615         'dump_intermediate_pages': opts.dump_intermediate_pages,
 616         'test': opts.test,
 617         'keepvideo': opts.keepvideo,
 618         'min_filesize': opts.min_filesize,
 619         'max_filesize': opts.max_filesize,
 620         'daterange': date,
 621         'cachedir': opts.cachedir,
 622         'youtube_print_sig_code': opts.youtube_print_sig_code,
 623         'age_limit': opts.age_limit,
 624         'download_archive': opts.download_archive,
 625         })
 626
 627     if opts.verbose:
 628         write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
 629         try:
 630             sp = subprocess.Popen(
 631                 ['git', 'rev-parse', '--short', 'HEAD'],
 632                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 633                 cwd=os.path.dirname(os.path.abspath(__file__)))
 634             out, err = sp.communicate()
 635             out = out.decode().strip()
 636             if re.match('[0-9a-f]+', out):
 637                 write_string(u'[debug] Git HEAD: ' + out + u'\n')
 638         except:
 639             try:
 640                 sys.exc_clear()
 641             except:
 642                 pass
 643         write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
 644
 645         proxy_map = {}
 646         for handler in opener.handlers:
 647             if hasattr(handler, 'proxies'):
 648                 proxy_map.update(handler.proxies)
 649         write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
 650
 651     ydl.add_default_info_extractors()
 652
 653     # PostProcessors
 654     if opts.extractaudio:
 655         ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 656     if opts.recodevideo:
 657         ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 658     if opts.embedsubtitles:
 659         ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 660
 661     # Update version
 662     if opts.update_self:
 663         update_self(ydl.to_screen, opts.verbose)
 664
 665     # Maybe do nothing
 666     if len(all_urls) < 1:
 667         if not opts.update_self:
 668             parser.error(u'you must provide at least one URL')
 669         else:
 670             sys.exit()
 671
 672     try:
 673         retcode = ydl.download(all_urls)
 674     except MaxDownloadsReached:
 675         ydl.to_screen(u'--max-download limit reached, aborting.')
 676         retcode = 101
 677
 678     # Dump cookie jar if requested
 679     if opts.cookiefile is not None:
 680         try:
 681             jar.save()
 682         except (IOError, OSError) as err:
 683             sys.exit(u'ERROR: unable to save cookie jar')
 684
 685     sys.exit(retcode)
 686
 687
 688 def _setup_opener(jar=None, opts=None, timeout=300):
 689     if opts is None:
 690         FakeOptions = collections.namedtuple(
 691             'FakeOptions', ['proxy', 'no_check_certificate'])
 692         opts = FakeOptions(proxy=None, no_check_certificate=False)
 693
 694     cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
 695     if opts.proxy is not None:
 696         if opts.proxy == '':
 697             proxies = {}
 698         else:
 699             proxies = {'http': opts.proxy, 'https': opts.proxy}
 700     else:
 701         proxies = compat_urllib_request.getproxies()
 702         # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
 703         if 'http' in proxies and 'https' not in proxies:
 704             proxies['https'] = proxies['http']
 705     proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 706     https_handler = make_HTTPS_handler(opts)
 707     opener = compat_urllib_request.build_opener(
 708         https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
 709     # Delete the default user-agent header, which would otherwise apply in
 710     # cases where our custom HTTP handler doesn't come into play
 711     # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
 712     opener.addheaders = []
 713     compat_urllib_request.install_opener(opener)
 714     socket.setdefaulttimeout(timeout)
 715     return opener
 716
 717
 718 def main(argv=None):
 719     try:
 720         _real_main(argv)
 721     except DownloadError:
 722         sys.exit(1)
 723     except SameFileError:
 724         sys.exit(u'ERROR: fixed output name but more than one file to download')
 725     except KeyboardInterrupt:
 726         sys.exit(u'\nERROR: Interrupted by user')