youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41 )
  42
  43 __license__ = 'Public Domain'
  44
  45 import codecs
  46 import getpass
  47 import locale
  48 import optparse
  49 import os
  50 import random
  51 import re
  52 import shlex
  53 import sys
  54
  55
  56 from .utils import (
  57     compat_print,
  58     DateRange,
  59     decodeOption,
  60     get_term_width,
  61     DownloadError,
  62     get_cachedir,
  63     MaxDownloadsReached,
  64     preferredencoding,
  65     SameFileError,
  66     setproctitle,
  67     std_headers,
  68     write_string,
  69 )
  70 from .update import update_self
  71 from .FileDownloader import (
  72     FileDownloader,
  73 )
  74 from .extractor import gen_extractors
  75 from .version import __version__
  76 from .YoutubeDL import YoutubeDL
  77 from .PostProcessor import (
  78     FFmpegMetadataPP,
  79     FFmpegVideoConvertor,
  80     FFmpegExtractAudioPP,
  81     FFmpegEmbedSubtitlePP,
  82 )
  83
  84
  85 def parseOpts(overrideArguments=None):
  86     def _readOptions(filename_bytes, default=[]):
  87         try:
  88             optionf = open(filename_bytes)
  89         except IOError:
  90             return default  # silently skip if file is not present
  91         try:
  92             res = []
  93             for l in optionf:
  94                 res += shlex.split(l, comments=True)
  95         finally:
  96             optionf.close()
  97         return res
  98
  99     def _format_option_string(option):
 100         ''' ('-o', '--option') -> -o, --format METAVAR'''
 101
 102         opts = []
 103
 104         if option._short_opts:
 105             opts.append(option._short_opts[0])
 106         if option._long_opts:
 107             opts.append(option._long_opts[0])
 108         if len(opts) > 1:
 109             opts.insert(1, ', ')
 110
 111         if option.takes_value(): opts.append(' %s' % option.metavar)
 112
 113         return "".join(opts)
 114
 115     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 116         setattr(parser.values, option.dest, value.split(','))
 117
 118     def _hide_login_info(opts):
 119         opts = list(opts)
 120         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 121             try:
 122                 i = opts.index(private_opt)
 123                 opts[i+1] = '<PRIVATE>'
 124             except ValueError:
 125                 pass
 126         return opts
 127
 128     max_width = 80
 129     max_help_position = 80
 130
 131     # No need to wrap help messages if we're on a wide console
 132     columns = get_term_width()
 133     if columns: max_width = columns
 134
 135     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 136     fmt.format_option_strings = _format_option_string
 137
 138     kw = {
 139         'version'   : __version__,
 140         'formatter' : fmt,
 141         'usage' : '%prog [options] url [url...]',
 142         'conflict_handler' : 'resolve',
 143     }
 144
 145     parser = optparse.OptionParser(**kw)
 146
 147     # option groups
 148     general        = optparse.OptionGroup(parser, 'General Options')
 149     selection      = optparse.OptionGroup(parser, 'Video Selection')
 150     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 151     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 152     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 153     downloader     = optparse.OptionGroup(parser, 'Download Options')
 154     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 155     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 156     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 157
 158     general.add_option('-h', '--help',
 159             action='help', help='print this help text and exit')
 160     general.add_option('-v', '--version',
 161             action='version', help='print program version and exit')
 162     general.add_option('-U', '--update',
 163             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 164     general.add_option('-i', '--ignore-errors',
 165             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 166     general.add_option('--abort-on-error',
 167             action='store_false', dest='ignoreerrors',
 168             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 169     general.add_option('--dump-user-agent',
 170             action='store_true', dest='dump_user_agent',
 171             help='display the current browser identification', default=False)
 172     general.add_option('--user-agent',
 173             dest='user_agent', help='specify a custom user agent', metavar='UA')
 174     general.add_option('--referer',
 175             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 176             metavar='REF', default=None)
 177     general.add_option('--list-extractors',
 178             action='store_true', dest='list_extractors',
 179             help='List all supported extractors and the URLs they would handle', default=False)
 180     general.add_option('--extractor-descriptions',
 181             action='store_true', dest='list_extractor_descriptions',
 182             help='Output descriptions of all supported extractors', default=False)
 183     general.add_option(
 184         '--proxy', dest='proxy', default=None, metavar='URL',
 185         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 186     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 187     general.add_option(
 188         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 189         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 190     general.add_option(
 191         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 192         help='Disable filesystem caching')
 193     general.add_option(
 194         '--socket-timeout', dest='socket_timeout',
 195         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 196     general.add_option(
 197         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 198         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 199
 200
 201     selection.add_option(
 202         '--playlist-start',
 203         dest='playliststart', metavar='NUMBER', default=1, type=int,
 204         help='playlist video to start at (default is %default)')
 205     selection.add_option(
 206         '--playlist-end',
 207         dest='playlistend', metavar='NUMBER', default=None, type=int,
 208         help='playlist video to end at (default is last)')
 209     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 210     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 211     selection.add_option('--max-downloads', metavar='NUMBER',
 212                          dest='max_downloads', type=int, default=None,
 213                          help='Abort after downloading NUMBER files')
 214     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 215     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 216     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 217     selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
 218     selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 219     selection.add_option(
 220         '--min-views', metavar='COUNT', dest='min_views',
 221         default=None, type=int,
 222         help="Do not download any videos with less than COUNT views",)
 223     selection.add_option(
 224         '--max-views', metavar='COUNT', dest='max_views',
 225         default=None, type=int,
 226         help="Do not download any videos with more than COUNT views",)
 227     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 228     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 229                          help='download only videos suitable for the given age',
 230                          default=None, type=int)
 231     selection.add_option('--download-archive', metavar='FILE',
 232                          dest='download_archive',
 233                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 234
 235
 236     authentication.add_option('-u', '--username',
 237             dest='username', metavar='USERNAME', help='account username')
 238     authentication.add_option('-p', '--password',
 239             dest='password', metavar='PASSWORD', help='account password')
 240     authentication.add_option('-n', '--netrc',
 241             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 242     authentication.add_option('--video-password',
 243             dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)')
 244
 245
 246     video_format.add_option('-f', '--format',
 247             action='store', dest='format', metavar='FORMAT', default='best',
 248             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
 249     video_format.add_option('--all-formats',
 250             action='store_const', dest='format', help='download all available video formats', const='all')
 251     video_format.add_option('--prefer-free-formats',
 252             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 253     video_format.add_option('--max-quality',
 254             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 255     video_format.add_option('-F', '--list-formats',
 256             action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 257
 258     subtitles.add_option('--write-sub', '--write-srt',
 259             action='store_true', dest='writesubtitles',
 260             help='write subtitle file', default=False)
 261     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 262             action='store_true', dest='writeautomaticsub',
 263             help='write automatic subtitle file (youtube only)', default=False)
 264     subtitles.add_option('--all-subs',
 265             action='store_true', dest='allsubtitles',
 266             help='downloads all the available subtitles of the video', default=False)
 267     subtitles.add_option('--list-subs',
 268             action='store_true', dest='listsubtitles',
 269             help='lists all available subtitles for the video', default=False)
 270     subtitles.add_option('--sub-format',
 271             action='store', dest='subtitlesformat', metavar='FORMAT',
 272             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 273     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 274             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 275             default=[], callback=_comma_separated_values_options_callback,
 276             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 277
 278     downloader.add_option('-r', '--rate-limit',
 279             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 280     downloader.add_option('-R', '--retries',
 281             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 282     downloader.add_option('--buffer-size',
 283             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 284     downloader.add_option('--no-resize-buffer',
 285             action='store_true', dest='noresizebuffer',
 286             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 287     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 288
 289     verbosity.add_option('-q', '--quiet',
 290             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 291     verbosity.add_option('-s', '--simulate',
 292             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 293     verbosity.add_option('--skip-download',
 294             action='store_true', dest='skip_download', help='do not download the video', default=False)
 295     verbosity.add_option('-g', '--get-url',
 296             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 297     verbosity.add_option('-e', '--get-title',
 298             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 299     verbosity.add_option('--get-id',
 300             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 301     verbosity.add_option('--get-thumbnail',
 302             action='store_true', dest='getthumbnail',
 303             help='simulate, quiet but print thumbnail URL', default=False)
 304     verbosity.add_option('--get-description',
 305             action='store_true', dest='getdescription',
 306             help='simulate, quiet but print video description', default=False)
 307     verbosity.add_option('--get-duration',
 308             action='store_true', dest='getduration',
 309             help='simulate, quiet but print video length', default=False)
 310     verbosity.add_option('--get-filename',
 311             action='store_true', dest='getfilename',
 312             help='simulate, quiet but print output filename', default=False)
 313     verbosity.add_option('--get-format',
 314             action='store_true', dest='getformat',
 315             help='simulate, quiet but print output format', default=False)
 316     verbosity.add_option('-j', '--dump-json',
 317             action='store_true', dest='dumpjson',
 318             help='simulate, quiet but print JSON information', default=False)
 319     verbosity.add_option('--newline',
 320             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 321     verbosity.add_option('--no-progress',
 322             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 323     verbosity.add_option('--console-title',
 324             action='store_true', dest='consoletitle',
 325             help='display progress in console titlebar', default=False)
 326     verbosity.add_option('-v', '--verbose',
 327             action='store_true', dest='verbose', help='print various debugging information', default=False)
 328     verbosity.add_option('--dump-intermediate-pages',
 329             action='store_true', dest='dump_intermediate_pages', default=False,
 330             help='print downloaded pages to debug problems(very verbose)')
 331     verbosity.add_option('--write-pages',
 332             action='store_true', dest='write_pages', default=False,
 333             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 334     verbosity.add_option('--youtube-print-sig-code',
 335             action='store_true', dest='youtube_print_sig_code', default=False,
 336             help=optparse.SUPPRESS_HELP)
 337     verbosity.add_option('--print-traffic',
 338             dest='debug_printtraffic', action='store_true', default=False,
 339             help=optparse.SUPPRESS_HELP)
 340
 341     filesystem.add_option('-t', '--title',
 342             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 343     filesystem.add_option('--id',
 344             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 345     filesystem.add_option('-l', '--literal',
 346             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 347     filesystem.add_option('-A', '--auto-number',
 348             action='store_true', dest='autonumber',
 349             help='number downloaded files starting from 00000', default=False)
 350     filesystem.add_option('-o', '--output',
 351             dest='outtmpl', metavar='TEMPLATE',
 352             help=('output filename template. Use %(title)s to get the title, '
 353                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 354                   '%(autonumber)s to get an automatically incremented number, '
 355                   '%(ext)s for the filename extension, '
 356                   '%(format)s for the format description (like "22 - 1280x720" or "HD"),'
 357                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"),'
 358                   '%(upload_date)s for the upload date (YYYYMMDD), '
 359                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 360                   '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
 361                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 362                   'Use - to output to stdout. Can also be used to download to a different directory, '
 363                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 364     filesystem.add_option('--autonumber-size',
 365             dest='autonumber_size', metavar='NUMBER',
 366             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 367     filesystem.add_option('--restrict-filenames',
 368             action='store_true', dest='restrictfilenames',
 369             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 370     filesystem.add_option('-a', '--batch-file',
 371             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 372     filesystem.add_option('--load-info',
 373             dest='load_info_filename', metavar='FILE',
 374             help='json file containing the video information (created with the "--write-json" option')
 375     filesystem.add_option('-w', '--no-overwrites',
 376             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 377     filesystem.add_option('-c', '--continue',
 378             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 379     filesystem.add_option('--no-continue',
 380             action='store_false', dest='continue_dl',
 381             help='do not resume partially downloaded files (restart from beginning)')
 382     filesystem.add_option('--cookies',
 383             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 384     filesystem.add_option('--no-part',
 385             action='store_true', dest='nopart', help='do not use .part files', default=False)
 386     filesystem.add_option('--no-mtime',
 387             action='store_false', dest='updatetime',
 388             help='do not use the Last-modified header to set the file modification time', default=True)
 389     filesystem.add_option('--write-description',
 390             action='store_true', dest='writedescription',
 391             help='write video description to a .description file', default=False)
 392     filesystem.add_option('--write-info-json',
 393             action='store_true', dest='writeinfojson',
 394             help='write video metadata to a .info.json file', default=False)
 395     filesystem.add_option('--write-annotations',
 396             action='store_true', dest='writeannotations',
 397             help='write video annotations to a .annotation file', default=False)
 398     filesystem.add_option('--write-thumbnail',
 399             action='store_true', dest='writethumbnail',
 400             help='write thumbnail image to disk', default=False)
 401
 402
 403     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 404             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 405     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 406             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 407     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 408             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 409     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 410             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 411     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 412             help='keeps the video file on disk after the post-processing; the video is erased by default')
 413     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 414             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 415     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 416             help='embed subtitles in the video (only for mp4 videos)')
 417     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 418             help='add metadata to the files')
 419
 420
 421     parser.add_option_group(general)
 422     parser.add_option_group(selection)
 423     parser.add_option_group(downloader)
 424     parser.add_option_group(filesystem)
 425     parser.add_option_group(verbosity)
 426     parser.add_option_group(video_format)
 427     parser.add_option_group(subtitles)
 428     parser.add_option_group(authentication)
 429     parser.add_option_group(postproc)
 430
 431     if overrideArguments is not None:
 432         opts, args = parser.parse_args(overrideArguments)
 433         if opts.verbose:
 434             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 435     else:
 436         systemConf = _readOptions('/etc/youtube-dl.conf')
 437
 438         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 439         if xdg_config_home:
 440             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 441             if not os.path.isfile(userConfFile):
 442                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 443         else:
 444             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 445             if not os.path.isfile(userConfFile):
 446                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 447         userConf = _readOptions(userConfFile, None)
 448
 449         if userConf is None:
 450             appdata_dir = os.environ.get('appdata')
 451             if appdata_dir:
 452                 userConf = _readOptions(
 453                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 454                     default=None)
 455                 if userConf is None:
 456                     userConf = _readOptions(
 457                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 458                         default=None)
 459
 460         if userConf is None:
 461             userConf = _readOptions(
 462                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 463                 default=None)
 464         if userConf is None:
 465             userConf = _readOptions(
 466                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 467                 default=None)
 468
 469         if userConf is None:
 470             userConf = []
 471
 472         commandLineConf = sys.argv[1:]
 473         argv = systemConf + userConf + commandLineConf
 474         opts, args = parser.parse_args(argv)
 475         if opts.verbose:
 476             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 477             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 478             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 479             write_string(u'[debug] Encodings: locale %r, fs %r, out %r, pref: %r\n' %
 480                          (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, preferredencoding()))
 481
 482     return parser, opts, args
 483
 484
 485 def _real_main(argv=None):
 486     # Compatibility fixes for Windows
 487     if sys.platform == 'win32':
 488         # https://github.com/rg3/youtube-dl/issues/820
 489         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 490
 491     setproctitle(u'youtube-dl')
 492
 493     parser, opts, args = parseOpts(argv)
 494
 495     # Set user agent
 496     if opts.user_agent is not None:
 497         std_headers['User-Agent'] = opts.user_agent
 498
 499     # Set referer
 500     if opts.referer is not None:
 501         std_headers['Referer'] = opts.referer
 502
 503     # Dump user agent
 504     if opts.dump_user_agent:
 505         compat_print(std_headers['User-Agent'])
 506         sys.exit(0)
 507
 508     # Batch file verification
 509     batchurls = []
 510     if opts.batchfile is not None:
 511         try:
 512             if opts.batchfile == '-':
 513                 batchfd = sys.stdin
 514             else:
 515                 batchfd = open(opts.batchfile, 'r')
 516             batchurls = batchfd.readlines()
 517             batchurls = [x.strip() for x in batchurls]
 518             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 519             if opts.verbose:
 520                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 521         except IOError:
 522             sys.exit(u'ERROR: batch file could not be read')
 523     all_urls = batchurls + args
 524     all_urls = [url.strip() for url in all_urls]
 525     _enc = preferredencoding()
 526     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url]
 527
 528     extractors = gen_extractors()
 529
 530     if opts.list_extractors:
 531         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 532             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 533             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 534             for mu in matchedUrls:
 535                 compat_print(u'  ' + mu)
 536         sys.exit(0)
 537     if opts.list_extractor_descriptions:
 538         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 539             if not ie._WORKING:
 540                 continue
 541             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 542             if desc is False:
 543                 continue
 544             if hasattr(ie, 'SEARCH_KEY'):
 545                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 546                 _COUNTS = (u'', u'5', u'10', u'all')
 547                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 548             compat_print(desc)
 549         sys.exit(0)
 550
 551
 552     # Conflicting, missing and erroneous options
 553     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 554         parser.error(u'using .netrc conflicts with giving username/password')
 555     if opts.password is not None and opts.username is None:
 556         parser.error(u' account username missing\n')
 557     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 558         parser.error(u'using output template conflicts with using title, video ID or auto number')
 559     if opts.usetitle and opts.useid:
 560         parser.error(u'using title conflicts with using video ID')
 561     if opts.username is not None and opts.password is None:
 562         opts.password = getpass.getpass(u'Type account password and press return:')
 563     if opts.ratelimit is not None:
 564         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 565         if numeric_limit is None:
 566             parser.error(u'invalid rate limit specified')
 567         opts.ratelimit = numeric_limit
 568     if opts.min_filesize is not None:
 569         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 570         if numeric_limit is None:
 571             parser.error(u'invalid min_filesize specified')
 572         opts.min_filesize = numeric_limit
 573     if opts.max_filesize is not None:
 574         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 575         if numeric_limit is None:
 576             parser.error(u'invalid max_filesize specified')
 577         opts.max_filesize = numeric_limit
 578     if opts.retries is not None:
 579         try:
 580             opts.retries = int(opts.retries)
 581         except (TypeError, ValueError):
 582             parser.error(u'invalid retry count specified')
 583     if opts.buffersize is not None:
 584         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 585         if numeric_buffersize is None:
 586             parser.error(u'invalid buffer size specified')
 587         opts.buffersize = numeric_buffersize
 588     if opts.playliststart <= 0:
 589         raise ValueError(u'Playlist start must be positive')
 590     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 591         raise ValueError(u'Playlist end must be greater than playlist start')
 592     if opts.extractaudio:
 593         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 594             parser.error(u'invalid audio format specified')
 595     if opts.audioquality:
 596         opts.audioquality = opts.audioquality.strip('k').strip('K')
 597         if not opts.audioquality.isdigit():
 598             parser.error(u'invalid audio quality specified')
 599     if opts.recodevideo is not None:
 600         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 601             parser.error(u'invalid video recode format specified')
 602     if opts.date is not None:
 603         date = DateRange.day(opts.date)
 604     else:
 605         date = DateRange(opts.dateafter, opts.datebefore)
 606
 607     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 608     # this was the old behaviour if only --all-sub was given.
 609     if opts.allsubtitles and (opts.writeautomaticsub == False):
 610         opts.writesubtitles = True
 611
 612     if sys.version_info < (3,):
 613         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 614         if opts.outtmpl is not None:
 615             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 616     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 617             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 618             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 619             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 620             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 621             or (opts.useid and u'%(id)s.%(ext)s')
 622             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 623             or u'%(title)s-%(id)s.%(ext)s')
 624     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 625         parser.error(u'Cannot download a video and extract audio into the same'
 626                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 627                      u' template'.format(outtmpl))
 628
 629     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 630
 631     ydl_opts = {
 632         'usenetrc': opts.usenetrc,
 633         'username': opts.username,
 634         'password': opts.password,
 635         'videopassword': opts.videopassword,
 636         'quiet': (opts.quiet or any_printing),
 637         'forceurl': opts.geturl,
 638         'forcetitle': opts.gettitle,
 639         'forceid': opts.getid,
 640         'forcethumbnail': opts.getthumbnail,
 641         'forcedescription': opts.getdescription,
 642         'forceduration': opts.getduration,
 643         'forcefilename': opts.getfilename,
 644         'forceformat': opts.getformat,
 645         'forcejson': opts.dumpjson,
 646         'simulate': opts.simulate,
 647         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 648         'format': opts.format,
 649         'format_limit': opts.format_limit,
 650         'listformats': opts.listformats,
 651         'outtmpl': outtmpl,
 652         'autonumber_size': opts.autonumber_size,
 653         'restrictfilenames': opts.restrictfilenames,
 654         'ignoreerrors': opts.ignoreerrors,
 655         'ratelimit': opts.ratelimit,
 656         'nooverwrites': opts.nooverwrites,
 657         'retries': opts.retries,
 658         'buffersize': opts.buffersize,
 659         'noresizebuffer': opts.noresizebuffer,
 660         'continuedl': opts.continue_dl,
 661         'noprogress': opts.noprogress,
 662         'progress_with_newline': opts.progress_with_newline,
 663         'playliststart': opts.playliststart,
 664         'playlistend': opts.playlistend,
 665         'noplaylist': opts.noplaylist,
 666         'logtostderr': opts.outtmpl == '-',
 667         'consoletitle': opts.consoletitle,
 668         'nopart': opts.nopart,
 669         'updatetime': opts.updatetime,
 670         'writedescription': opts.writedescription,
 671         'writeannotations': opts.writeannotations,
 672         'writeinfojson': opts.writeinfojson,
 673         'writethumbnail': opts.writethumbnail,
 674         'writesubtitles': opts.writesubtitles,
 675         'writeautomaticsub': opts.writeautomaticsub,
 676         'allsubtitles': opts.allsubtitles,
 677         'listsubtitles': opts.listsubtitles,
 678         'subtitlesformat': opts.subtitlesformat,
 679         'subtitleslangs': opts.subtitleslangs,
 680         'matchtitle': decodeOption(opts.matchtitle),
 681         'rejecttitle': decodeOption(opts.rejecttitle),
 682         'max_downloads': opts.max_downloads,
 683         'prefer_free_formats': opts.prefer_free_formats,
 684         'verbose': opts.verbose,
 685         'dump_intermediate_pages': opts.dump_intermediate_pages,
 686         'write_pages': opts.write_pages,
 687         'test': opts.test,
 688         'keepvideo': opts.keepvideo,
 689         'min_filesize': opts.min_filesize,
 690         'max_filesize': opts.max_filesize,
 691         'min_views': opts.min_views,
 692         'max_views': opts.max_views,
 693         'daterange': date,
 694         'cachedir': opts.cachedir,
 695         'youtube_print_sig_code': opts.youtube_print_sig_code,
 696         'age_limit': opts.age_limit,
 697         'download_archive': opts.download_archive,
 698         'cookiefile': opts.cookiefile,
 699         'nocheckcertificate': opts.no_check_certificate,
 700         'proxy': opts.proxy,
 701         'socket_timeout': opts.socket_timeout,
 702         'bidi_workaround': opts.bidi_workaround,
 703         'debug_printtraffic': opts.debug_printtraffic,
 704     }
 705
 706     with YoutubeDL(ydl_opts) as ydl:
 707         ydl.print_debug_header()
 708         ydl.add_default_info_extractors()
 709
 710         # PostProcessors
 711         # Add the metadata pp first, the other pps will copy it
 712         if opts.addmetadata:
 713             ydl.add_post_processor(FFmpegMetadataPP())
 714         if opts.extractaudio:
 715             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 716         if opts.recodevideo:
 717             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 718         if opts.embedsubtitles:
 719             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 720
 721         # Update version
 722         if opts.update_self:
 723             update_self(ydl.to_screen, opts.verbose)
 724
 725         # Maybe do nothing
 726         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 727             if not opts.update_self:
 728                 parser.error(u'you must provide at least one URL')
 729             else:
 730                 sys.exit()
 731
 732         try:
 733             if opts.load_info_filename is not None:
 734                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 735             else:
 736                 retcode = ydl.download(all_urls)
 737         except MaxDownloadsReached:
 738             ydl.to_screen(u'--max-download limit reached, aborting.')
 739             retcode = 101
 740
 741     sys.exit(retcode)
 742
 743
 744 def main(argv=None):
 745     try:
 746         _real_main(argv)
 747     except DownloadError:
 748         sys.exit(1)
 749     except SameFileError:
 750         sys.exit(u'ERROR: fixed output name but more than one file to download')
 751     except KeyboardInterrupt:
 752         sys.exit(u'\nERROR: Interrupted by user')