youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35 )
  36
  37 __license__ = 'Public Domain'
  38
  39 import codecs
  40 import collections
  41 import getpass
  42 import optparse
  43 import os
  44 import random
  45 import re
  46 import shlex
  47 import socket
  48 import subprocess
  49 import sys
  50 import traceback
  51 import platform
  52
  53
  54 from .utils import (
  55     compat_cookiejar,
  56     compat_print,
  57     compat_str,
  58     compat_urllib_request,
  59     DateRange,
  60     decodeOption,
  61     determine_ext,
  62     DownloadError,
  63     get_cachedir,
  64     make_HTTPS_handler,
  65     MaxDownloadsReached,
  66     platform_name,
  67     preferredencoding,
  68     SameFileError,
  69     std_headers,
  70     write_string,
  71     YoutubeDLHandler,
  72 )
  73 from .update import update_self
  74 from .version import __version__
  75 from .FileDownloader import (
  76     FileDownloader,
  77 )
  78 from .extractor import gen_extractors
  79 from .YoutubeDL import YoutubeDL
  80 from .PostProcessor import (
  81     FFmpegMetadataPP,
  82     FFmpegVideoConvertor,
  83     FFmpegExtractAudioPP,
  84     FFmpegEmbedSubtitlePP,
  85 )
  86
  87
  88 def parseOpts(overrideArguments=None):
  89     def _readOptions(filename_bytes):
  90         try:
  91             optionf = open(filename_bytes)
  92         except IOError:
  93             return [] # silently skip if file is not present
  94         try:
  95             res = []
  96             for l in optionf:
  97                 res += shlex.split(l, comments=True)
  98         finally:
  99             optionf.close()
 100         return res
 101
 102     def _format_option_string(option):
 103         ''' ('-o', '--option') -> -o, --format METAVAR'''
 104
 105         opts = []
 106
 107         if option._short_opts:
 108             opts.append(option._short_opts[0])
 109         if option._long_opts:
 110             opts.append(option._long_opts[0])
 111         if len(opts) > 1:
 112             opts.insert(1, ', ')
 113
 114         if option.takes_value(): opts.append(' %s' % option.metavar)
 115
 116         return "".join(opts)
 117
 118     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 119         setattr(parser.values, option.dest, value.split(','))
 120
 121     def _find_term_columns():
 122         columns = os.environ.get('COLUMNS', None)
 123         if columns:
 124             return int(columns)
 125
 126         try:
 127             sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 128             out,err = sp.communicate()
 129             return int(out.split()[1])
 130         except:
 131             pass
 132         return None
 133
 134     def _hide_login_info(opts):
 135         opts = list(opts)
 136         for private_opt in ['-p', '--password', '-u', '--username']:
 137             try:
 138                 i = opts.index(private_opt)
 139                 opts[i+1] = '<PRIVATE>'
 140             except ValueError:
 141                 pass
 142         return opts
 143
 144     max_width = 80
 145     max_help_position = 80
 146
 147     # No need to wrap help messages if we're on a wide console
 148     columns = _find_term_columns()
 149     if columns: max_width = columns
 150
 151     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 152     fmt.format_option_strings = _format_option_string
 153
 154     kw = {
 155         'version'   : __version__,
 156         'formatter' : fmt,
 157         'usage' : '%prog [options] url [url...]',
 158         'conflict_handler' : 'resolve',
 159     }
 160
 161     parser = optparse.OptionParser(**kw)
 162
 163     # option groups
 164     general        = optparse.OptionGroup(parser, 'General Options')
 165     selection      = optparse.OptionGroup(parser, 'Video Selection')
 166     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 167     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 168     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 169     downloader     = optparse.OptionGroup(parser, 'Download Options')
 170     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 171     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 172     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 173
 174     general.add_option('-h', '--help',
 175             action='help', help='print this help text and exit')
 176     general.add_option('-v', '--version',
 177             action='version', help='print program version and exit')
 178     general.add_option('-U', '--update',
 179             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 180     general.add_option('-i', '--ignore-errors',
 181             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 182     general.add_option('--dump-user-agent',
 183             action='store_true', dest='dump_user_agent',
 184             help='display the current browser identification', default=False)
 185     general.add_option('--user-agent',
 186             dest='user_agent', help='specify a custom user agent', metavar='UA')
 187     general.add_option('--referer',
 188             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 189             metavar='REF', default=None)
 190     general.add_option('--list-extractors',
 191             action='store_true', dest='list_extractors',
 192             help='List all supported extractors and the URLs they would handle', default=False)
 193     general.add_option('--extractor-descriptions',
 194             action='store_true', dest='list_extractor_descriptions',
 195             help='Output descriptions of all supported extractors', default=False)
 196     general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
 197     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 198     general.add_option(
 199         '--cache-dir', dest='cachedir', default=get_cachedir(),
 200         help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
 201     general.add_option(
 202         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 203         help='Disable filesystem caching')
 204
 205
 206     selection.add_option('--playlist-start',
 207             dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
 208     selection.add_option('--playlist-end',
 209             dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
 210     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 211     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 212     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
 213     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 214     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 215     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 216     selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
 217     selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 218     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 219     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 220                          help='download only videos suitable for the given age',
 221                          default=None, type=int)
 222     selection.add_option('--download-archive', metavar='FILE',
 223                          dest='download_archive',
 224                          help='Download only videos not present in the archive file. Record all downloaded videos in it.')
 225
 226
 227     authentication.add_option('-u', '--username',
 228             dest='username', metavar='USERNAME', help='account username')
 229     authentication.add_option('-p', '--password',
 230             dest='password', metavar='PASSWORD', help='account password')
 231     authentication.add_option('-n', '--netrc',
 232             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 233     authentication.add_option('--video-password',
 234             dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)')
 235
 236
 237     video_format.add_option('-f', '--format',
 238             action='store', dest='format', metavar='FORMAT', default='best',
 239             help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
 240     video_format.add_option('--all-formats',
 241             action='store_const', dest='format', help='download all available video formats', const='all')
 242     video_format.add_option('--prefer-free-formats',
 243             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 244     video_format.add_option('--max-quality',
 245             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 246     video_format.add_option('-F', '--list-formats',
 247             action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 248
 249     subtitles.add_option('--write-sub', '--write-srt',
 250             action='store_true', dest='writesubtitles',
 251             help='write subtitle file', default=False)
 252     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 253             action='store_true', dest='writeautomaticsub',
 254             help='write automatic subtitle file (youtube only)', default=False)
 255     subtitles.add_option('--all-subs',
 256             action='store_true', dest='allsubtitles',
 257             help='downloads all the available subtitles of the video', default=False)
 258     subtitles.add_option('--list-subs',
 259             action='store_true', dest='listsubtitles',
 260             help='lists all available subtitles for the video', default=False)
 261     subtitles.add_option('--sub-format',
 262             action='store', dest='subtitlesformat', metavar='FORMAT',
 263             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 264     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 265             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 266             default=[], callback=_comma_separated_values_options_callback,
 267             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 268
 269     downloader.add_option('-r', '--rate-limit',
 270             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 271     downloader.add_option('-R', '--retries',
 272             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 273     downloader.add_option('--buffer-size',
 274             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 275     downloader.add_option('--no-resize-buffer',
 276             action='store_true', dest='noresizebuffer',
 277             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 278     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 279
 280     verbosity.add_option('-q', '--quiet',
 281             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 282     verbosity.add_option('-s', '--simulate',
 283             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 284     verbosity.add_option('--skip-download',
 285             action='store_true', dest='skip_download', help='do not download the video', default=False)
 286     verbosity.add_option('-g', '--get-url',
 287             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 288     verbosity.add_option('-e', '--get-title',
 289             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 290     verbosity.add_option('--get-id',
 291             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 292     verbosity.add_option('--get-thumbnail',
 293             action='store_true', dest='getthumbnail',
 294             help='simulate, quiet but print thumbnail URL', default=False)
 295     verbosity.add_option('--get-description',
 296             action='store_true', dest='getdescription',
 297             help='simulate, quiet but print video description', default=False)
 298     verbosity.add_option('--get-filename',
 299             action='store_true', dest='getfilename',
 300             help='simulate, quiet but print output filename', default=False)
 301     verbosity.add_option('--get-format',
 302             action='store_true', dest='getformat',
 303             help='simulate, quiet but print output format', default=False)
 304     verbosity.add_option('--newline',
 305             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 306     verbosity.add_option('--no-progress',
 307             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 308     verbosity.add_option('--console-title',
 309             action='store_true', dest='consoletitle',
 310             help='display progress in console titlebar', default=False)
 311     verbosity.add_option('-v', '--verbose',
 312             action='store_true', dest='verbose', help='print various debugging information', default=False)
 313     verbosity.add_option('--dump-intermediate-pages',
 314             action='store_true', dest='dump_intermediate_pages', default=False,
 315             help='print downloaded pages to debug problems(very verbose)')
 316     verbosity.add_option('--youtube-print-sig-code',
 317             action='store_true', dest='youtube_print_sig_code', default=False,
 318             help=optparse.SUPPRESS_HELP)
 319
 320
 321     filesystem.add_option('-t', '--title',
 322             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 323     filesystem.add_option('--id',
 324             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 325     filesystem.add_option('-l', '--literal',
 326             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 327     filesystem.add_option('-A', '--auto-number',
 328             action='store_true', dest='autonumber',
 329             help='number downloaded files starting from 00000', default=False)
 330     filesystem.add_option('-o', '--output',
 331             dest='outtmpl', metavar='TEMPLATE',
 332             help=('output filename template. Use %(title)s to get the title, '
 333                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 334                   '%(autonumber)s to get an automatically incremented number, '
 335                   '%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), '
 336                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 337                   '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
 338                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 339                   'Use - to output to stdout. Can also be used to download to a different directory, '
 340                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 341     filesystem.add_option('--autonumber-size',
 342             dest='autonumber_size', metavar='NUMBER',
 343             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given')
 344     filesystem.add_option('--restrict-filenames',
 345             action='store_true', dest='restrictfilenames',
 346             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 347     filesystem.add_option('-a', '--batch-file',
 348             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 349     filesystem.add_option('-w', '--no-overwrites',
 350             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 351     filesystem.add_option('-c', '--continue',
 352             action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
 353     filesystem.add_option('--no-continue',
 354             action='store_false', dest='continue_dl',
 355             help='do not resume partially downloaded files (restart from beginning)')
 356     filesystem.add_option('--cookies',
 357             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 358     filesystem.add_option('--no-part',
 359             action='store_true', dest='nopart', help='do not use .part files', default=False)
 360     filesystem.add_option('--no-mtime',
 361             action='store_false', dest='updatetime',
 362             help='do not use the Last-modified header to set the file modification time', default=True)
 363     filesystem.add_option('--write-description',
 364             action='store_true', dest='writedescription',
 365             help='write video description to a .description file', default=False)
 366     filesystem.add_option('--write-info-json',
 367             action='store_true', dest='writeinfojson',
 368             help='write video metadata to a .info.json file', default=False)
 369     filesystem.add_option('--write-annotations',
 370             action='store_true', dest='writeannotations',
 371             help='write video annotations to a .annotation file', default=False)
 372     filesystem.add_option('--write-thumbnail',
 373             action='store_true', dest='writethumbnail',
 374             help='write thumbnail image to disk', default=False)
 375
 376
 377     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 378             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 379     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 380             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 381     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 382             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 383     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 384             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 385     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 386             help='keeps the video file on disk after the post-processing; the video is erased by default')
 387     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 388             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 389     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 390             help='embed subtitles in the video (only for mp4 videos)')
 391     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 392             help='add metadata to the files')
 393
 394
 395     parser.add_option_group(general)
 396     parser.add_option_group(selection)
 397     parser.add_option_group(downloader)
 398     parser.add_option_group(filesystem)
 399     parser.add_option_group(verbosity)
 400     parser.add_option_group(video_format)
 401     parser.add_option_group(subtitles)
 402     parser.add_option_group(authentication)
 403     parser.add_option_group(postproc)
 404
 405     if overrideArguments is not None:
 406         opts, args = parser.parse_args(overrideArguments)
 407         if opts.verbose:
 408             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 409     else:
 410         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 411         if xdg_config_home:
 412             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 413             if not os.path.isfile(userConfFile):
 414                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 415         else:
 416             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 417             if not os.path.isfile(userConfFile):
 418                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 419         systemConf = _readOptions('/etc/youtube-dl.conf')
 420         userConf = _readOptions(userConfFile)
 421         commandLineConf = sys.argv[1:]
 422         argv = systemConf + userConf + commandLineConf
 423         opts, args = parser.parse_args(argv)
 424         if opts.verbose:
 425             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 426             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 427             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 428
 429     return parser, opts, args
 430
 431 def _real_main(argv=None):
 432     # Compatibility fixes for Windows
 433     if sys.platform == 'win32':
 434         # https://github.com/rg3/youtube-dl/issues/820
 435         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 436
 437     parser, opts, args = parseOpts(argv)
 438
 439     # Open appropriate CookieJar
 440     if opts.cookiefile is None:
 441         jar = compat_cookiejar.CookieJar()
 442     else:
 443         try:
 444             jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
 445             if os.access(opts.cookiefile, os.R_OK):
 446                 jar.load()
 447         except (IOError, OSError) as err:
 448             if opts.verbose:
 449                 traceback.print_exc()
 450             write_string(u'ERROR: unable to open cookie file\n')
 451             sys.exit(101)
 452     # Set user agent
 453     if opts.user_agent is not None:
 454         std_headers['User-Agent'] = opts.user_agent
 455
 456     # Set referer
 457     if opts.referer is not None:
 458         std_headers['Referer'] = opts.referer
 459
 460     # Dump user agent
 461     if opts.dump_user_agent:
 462         compat_print(std_headers['User-Agent'])
 463         sys.exit(0)
 464
 465     # Batch file verification
 466     batchurls = []
 467     if opts.batchfile is not None:
 468         try:
 469             if opts.batchfile == '-':
 470                 batchfd = sys.stdin
 471             else:
 472                 batchfd = open(opts.batchfile, 'r')
 473             batchurls = batchfd.readlines()
 474             batchurls = [x.strip() for x in batchurls]
 475             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 476             if opts.verbose:
 477                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 478         except IOError:
 479             sys.exit(u'ERROR: batch file could not be read')
 480     all_urls = batchurls + args
 481     all_urls = [url.strip() for url in all_urls]
 482
 483     opener = _setup_opener(jar=jar, opts=opts)
 484
 485     extractors = gen_extractors()
 486
 487     if opts.list_extractors:
 488         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 489             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 490             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 491             all_urls = [url for url in all_urls if url not in matchedUrls]
 492             for mu in matchedUrls:
 493                 compat_print(u'  ' + mu)
 494         sys.exit(0)
 495     if opts.list_extractor_descriptions:
 496         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 497             if not ie._WORKING:
 498                 continue
 499             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 500             if desc is False:
 501                 continue
 502             if hasattr(ie, 'SEARCH_KEY'):
 503                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 504                 _COUNTS = (u'', u'5', u'10', u'all')
 505                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 506             compat_print(desc)
 507         sys.exit(0)
 508
 509
 510     # Conflicting, missing and erroneous options
 511     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 512         parser.error(u'using .netrc conflicts with giving username/password')
 513     if opts.password is not None and opts.username is None:
 514         parser.error(u' account username missing\n')
 515     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 516         parser.error(u'using output template conflicts with using title, video ID or auto number')
 517     if opts.usetitle and opts.useid:
 518         parser.error(u'using title conflicts with using video ID')
 519     if opts.username is not None and opts.password is None:
 520         opts.password = getpass.getpass(u'Type account password and press return:')
 521     if opts.ratelimit is not None:
 522         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 523         if numeric_limit is None:
 524             parser.error(u'invalid rate limit specified')
 525         opts.ratelimit = numeric_limit
 526     if opts.min_filesize is not None:
 527         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 528         if numeric_limit is None:
 529             parser.error(u'invalid min_filesize specified')
 530         opts.min_filesize = numeric_limit
 531     if opts.max_filesize is not None:
 532         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 533         if numeric_limit is None:
 534             parser.error(u'invalid max_filesize specified')
 535         opts.max_filesize = numeric_limit
 536     if opts.retries is not None:
 537         try:
 538             opts.retries = int(opts.retries)
 539         except (TypeError, ValueError) as err:
 540             parser.error(u'invalid retry count specified')
 541     if opts.buffersize is not None:
 542         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 543         if numeric_buffersize is None:
 544             parser.error(u'invalid buffer size specified')
 545         opts.buffersize = numeric_buffersize
 546     try:
 547         opts.playliststart = int(opts.playliststart)
 548         if opts.playliststart <= 0:
 549             raise ValueError(u'Playlist start must be positive')
 550     except (TypeError, ValueError) as err:
 551         parser.error(u'invalid playlist start number specified')
 552     try:
 553         opts.playlistend = int(opts.playlistend)
 554         if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
 555             raise ValueError(u'Playlist end must be greater than playlist start')
 556     except (TypeError, ValueError) as err:
 557         parser.error(u'invalid playlist end number specified')
 558     if opts.extractaudio:
 559         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 560             parser.error(u'invalid audio format specified')
 561     if opts.audioquality:
 562         opts.audioquality = opts.audioquality.strip('k').strip('K')
 563         if not opts.audioquality.isdigit():
 564             parser.error(u'invalid audio quality specified')
 565     if opts.recodevideo is not None:
 566         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 567             parser.error(u'invalid video recode format specified')
 568     if opts.date is not None:
 569         date = DateRange.day(opts.date)
 570     else:
 571         date = DateRange(opts.dateafter, opts.datebefore)
 572
 573     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 574     # this was the old behaviour if only --all-sub was given.
 575     if opts.allsubtitles and (opts.writeautomaticsub == False):
 576         opts.writesubtitles = True
 577
 578     if sys.version_info < (3,):
 579         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 580         if opts.outtmpl is not None:
 581             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 582     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 583             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 584             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 585             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 586             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 587             or (opts.useid and u'%(id)s.%(ext)s')
 588             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 589             or u'%(title)s-%(id)s.%(ext)s')
 590     if '%(ext)s' not in outtmpl and opts.extractaudio:
 591         parser.error(u'Cannot download a video and extract audio into the same'
 592                      u' file! Use "%%(ext)s" instead of %r' %
 593                      determine_ext(outtmpl, u''))
 594
 595     # YoutubeDL
 596     ydl = YoutubeDL({
 597         'usenetrc': opts.usenetrc,
 598         'username': opts.username,
 599         'password': opts.password,
 600         'videopassword': opts.videopassword,
 601         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 602         'forceurl': opts.geturl,
 603         'forcetitle': opts.gettitle,
 604         'forceid': opts.getid,
 605         'forcethumbnail': opts.getthumbnail,
 606         'forcedescription': opts.getdescription,
 607         'forcefilename': opts.getfilename,
 608         'forceformat': opts.getformat,
 609         'simulate': opts.simulate,
 610         'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 611         'format': opts.format,
 612         'format_limit': opts.format_limit,
 613         'listformats': opts.listformats,
 614         'outtmpl': outtmpl,
 615         'autonumber_size': opts.autonumber_size,
 616         'restrictfilenames': opts.restrictfilenames,
 617         'ignoreerrors': opts.ignoreerrors,
 618         'ratelimit': opts.ratelimit,
 619         'nooverwrites': opts.nooverwrites,
 620         'retries': opts.retries,
 621         'buffersize': opts.buffersize,
 622         'noresizebuffer': opts.noresizebuffer,
 623         'continuedl': opts.continue_dl,
 624         'noprogress': opts.noprogress,
 625         'progress_with_newline': opts.progress_with_newline,
 626         'playliststart': opts.playliststart,
 627         'playlistend': opts.playlistend,
 628         'noplaylist': opts.noplaylist,
 629         'logtostderr': opts.outtmpl == '-',
 630         'consoletitle': opts.consoletitle,
 631         'nopart': opts.nopart,
 632         'updatetime': opts.updatetime,
 633         'writedescription': opts.writedescription,
 634         'writeannotations': opts.writeannotations,
 635         'writeinfojson': opts.writeinfojson,
 636         'writethumbnail': opts.writethumbnail,
 637         'writesubtitles': opts.writesubtitles,
 638         'writeautomaticsub': opts.writeautomaticsub,
 639         'allsubtitles': opts.allsubtitles,
 640         'listsubtitles': opts.listsubtitles,
 641         'subtitlesformat': opts.subtitlesformat,
 642         'subtitleslangs': opts.subtitleslangs,
 643         'matchtitle': decodeOption(opts.matchtitle),
 644         'rejecttitle': decodeOption(opts.rejecttitle),
 645         'max_downloads': opts.max_downloads,
 646         'prefer_free_formats': opts.prefer_free_formats,
 647         'verbose': opts.verbose,
 648         'dump_intermediate_pages': opts.dump_intermediate_pages,
 649         'test': opts.test,
 650         'keepvideo': opts.keepvideo,
 651         'min_filesize': opts.min_filesize,
 652         'max_filesize': opts.max_filesize,
 653         'daterange': date,
 654         'cachedir': opts.cachedir,
 655         'youtube_print_sig_code': opts.youtube_print_sig_code,
 656         'age_limit': opts.age_limit,
 657         'download_archive': opts.download_archive,
 658         })
 659
 660     if opts.verbose:
 661         write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
 662         try:
 663             sp = subprocess.Popen(
 664                 ['git', 'rev-parse', '--short', 'HEAD'],
 665                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 666                 cwd=os.path.dirname(os.path.abspath(__file__)))
 667             out, err = sp.communicate()
 668             out = out.decode().strip()
 669             if re.match('[0-9a-f]+', out):
 670                 write_string(u'[debug] Git HEAD: ' + out + u'\n')
 671         except:
 672             try:
 673                 sys.exc_clear()
 674             except:
 675                 pass
 676         write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
 677
 678         proxy_map = {}
 679         for handler in opener.handlers:
 680             if hasattr(handler, 'proxies'):
 681                 proxy_map.update(handler.proxies)
 682         write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
 683
 684     ydl.add_default_info_extractors()
 685
 686     # PostProcessors
 687     # Add the metadata pp first, the other pps will copy it
 688     if opts.addmetadata:
 689         ydl.add_post_processor(FFmpegMetadataPP())
 690     if opts.extractaudio:
 691         ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 692     if opts.recodevideo:
 693         ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 694     if opts.embedsubtitles:
 695         ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 696
 697     # Update version
 698     if opts.update_self:
 699         update_self(ydl.to_screen, opts.verbose)
 700
 701     # Maybe do nothing
 702     if len(all_urls) < 1:
 703         if not opts.update_self:
 704             parser.error(u'you must provide at least one URL')
 705         else:
 706             sys.exit()
 707
 708     try:
 709         retcode = ydl.download(all_urls)
 710     except MaxDownloadsReached:
 711         ydl.to_screen(u'--max-download limit reached, aborting.')
 712         retcode = 101
 713
 714     # Dump cookie jar if requested
 715     if opts.cookiefile is not None:
 716         try:
 717             jar.save()
 718         except (IOError, OSError):
 719             sys.exit(u'ERROR: unable to save cookie jar')
 720
 721     sys.exit(retcode)
 722
 723
 724 def _setup_opener(jar=None, opts=None, timeout=300):
 725     if opts is None:
 726         FakeOptions = collections.namedtuple(
 727             'FakeOptions', ['proxy', 'no_check_certificate'])
 728         opts = FakeOptions(proxy=None, no_check_certificate=False)
 729
 730     cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
 731     if opts.proxy is not None:
 732         if opts.proxy == '':
 733             proxies = {}
 734         else:
 735             proxies = {'http': opts.proxy, 'https': opts.proxy}
 736     else:
 737         proxies = compat_urllib_request.getproxies()
 738         # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
 739         if 'http' in proxies and 'https' not in proxies:
 740             proxies['https'] = proxies['http']
 741     proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 742     https_handler = make_HTTPS_handler(opts)
 743     opener = compat_urllib_request.build_opener(
 744         https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
 745     # Delete the default user-agent header, which would otherwise apply in
 746     # cases where our custom HTTP handler doesn't come into play
 747     # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
 748     opener.addheaders = []
 749     compat_urllib_request.install_opener(opener)
 750     socket.setdefaulttimeout(timeout)
 751     return opener
 752
 753
 754 def main(argv=None):
 755     try:
 756         _real_main(argv)
 757     except DownloadError:
 758         sys.exit(1)
 759     except SameFileError:
 760         sys.exit(u'ERROR: fixed output name but more than one file to download')
 761     except KeyboardInterrupt:
 762         sys.exit(u'\nERROR: Interrupted by user')