youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70     'David Fabijan',
  71     'Sebastian Haas',
  72     'Alexander Kirk',
  73     'Erik Johnson',
  74     'Keith Beckman',
  75     'Ole Ernst',
  76     'Aaron McDaniel (mcd1992)',
  77     'Magnus Kolstad',
  78 )
  79
  80 __license__ = 'Public Domain'
  81
  82 import codecs
  83 import io
  84 import os
  85 import random
  86 import sys
  87
  88
  89 from .options import (
  90     parseOpts,
  91 )
  92 from .utils import (
  93     compat_getpass,
  94     compat_print,
  95     DateRange,
  96     DEFAULT_OUTTMPL,
  97     decodeOption,
  98     DownloadError,
  99     MaxDownloadsReached,
 100     preferredencoding,
 101     read_batch_urls,
 102     SameFileError,
 103     setproctitle,
 104     std_headers,
 105     write_string,
 106 )
 107 from .update import update_self
 108 from .downloader import (
 109     FileDownloader,
 110 )
 111 from .extractor import gen_extractors
 112 from .YoutubeDL import YoutubeDL
 113 from .postprocessor import (
 114     AtomicParsleyPP,
 115     FFmpegAudioFixPP,
 116     FFmpegMetadataPP,
 117     FFmpegVideoConvertor,
 118     FFmpegExtractAudioPP,
 119     FFmpegEmbedSubtitlePP,
 120     XAttrMetadataPP,
 121     ExecAfterDownloadPP,
 122 )
 123
 124
 125 def _real_main(argv=None):
 126     # Compatibility fixes for Windows
 127     if sys.platform == 'win32':
 128         # https://github.com/rg3/youtube-dl/issues/820
 129         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 130
 131     setproctitle(u'youtube-dl')
 132
 133     parser, opts, args = parseOpts(argv)
 134
 135     # Set user agent
 136     if opts.user_agent is not None:
 137         std_headers['User-Agent'] = opts.user_agent
 138
 139     # Set referer
 140     if opts.referer is not None:
 141         std_headers['Referer'] = opts.referer
 142
 143     # Custom HTTP headers
 144     if opts.headers is not None:
 145         for h in opts.headers:
 146             if h.find(':', 1) < 0:
 147                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 148             key, value = h.split(':', 2)
 149             if opts.verbose:
 150                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 151             std_headers[key] = value
 152
 153     # Dump user agent
 154     if opts.dump_user_agent:
 155         compat_print(std_headers['User-Agent'])
 156         sys.exit(0)
 157
 158     # Batch file verification
 159     batch_urls = []
 160     if opts.batchfile is not None:
 161         try:
 162             if opts.batchfile == '-':
 163                 batchfd = sys.stdin
 164             else:
 165                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 166             batch_urls = read_batch_urls(batchfd)
 167             if opts.verbose:
 168                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 169         except IOError:
 170             sys.exit(u'ERROR: batch file could not be read')
 171     all_urls = batch_urls + args
 172     all_urls = [url.strip() for url in all_urls]
 173     _enc = preferredencoding()
 174     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 175
 176     extractors = gen_extractors()
 177
 178     if opts.list_extractors:
 179         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 180             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 181             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 182             for mu in matchedUrls:
 183                 compat_print(u'  ' + mu)
 184         sys.exit(0)
 185     if opts.list_extractor_descriptions:
 186         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 187             if not ie._WORKING:
 188                 continue
 189             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 190             if desc is False:
 191                 continue
 192             if hasattr(ie, 'SEARCH_KEY'):
 193                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 194                 _COUNTS = (u'', u'5', u'10', u'all')
 195                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 196             compat_print(desc)
 197         sys.exit(0)
 198
 199
 200     # Conflicting, missing and erroneous options
 201     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 202         parser.error(u'using .netrc conflicts with giving username/password')
 203     if opts.password is not None and opts.username is None:
 204         parser.error(u'account username missing\n')
 205     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 206         parser.error(u'using output template conflicts with using title, video ID or auto number')
 207     if opts.usetitle and opts.useid:
 208         parser.error(u'using title conflicts with using video ID')
 209     if opts.username is not None and opts.password is None:
 210         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 211     if opts.ratelimit is not None:
 212         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 213         if numeric_limit is None:
 214             parser.error(u'invalid rate limit specified')
 215         opts.ratelimit = numeric_limit
 216     if opts.min_filesize is not None:
 217         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 218         if numeric_limit is None:
 219             parser.error(u'invalid min_filesize specified')
 220         opts.min_filesize = numeric_limit
 221     if opts.max_filesize is not None:
 222         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 223         if numeric_limit is None:
 224             parser.error(u'invalid max_filesize specified')
 225         opts.max_filesize = numeric_limit
 226     if opts.retries is not None:
 227         try:
 228             opts.retries = int(opts.retries)
 229         except (TypeError, ValueError):
 230             parser.error(u'invalid retry count specified')
 231     if opts.buffersize is not None:
 232         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 233         if numeric_buffersize is None:
 234             parser.error(u'invalid buffer size specified')
 235         opts.buffersize = numeric_buffersize
 236     if opts.playliststart <= 0:
 237         raise ValueError(u'Playlist start must be positive')
 238     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 239         raise ValueError(u'Playlist end must be greater than playlist start')
 240     if opts.extractaudio:
 241         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 242             parser.error(u'invalid audio format specified')
 243     if opts.audioquality:
 244         opts.audioquality = opts.audioquality.strip('k').strip('K')
 245         if not opts.audioquality.isdigit():
 246             parser.error(u'invalid audio quality specified')
 247     if opts.recodevideo is not None:
 248         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 249             parser.error(u'invalid video recode format specified')
 250     if opts.date is not None:
 251         date = DateRange.day(opts.date)
 252     else:
 253         date = DateRange(opts.dateafter, opts.datebefore)
 254     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 255         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 256
 257     # Do not download videos when there are audio-only formats
 258     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 259         opts.format = 'bestaudio/best'
 260
 261     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 262     # this was the old behaviour if only --all-sub was given.
 263     if opts.allsubtitles and (opts.writeautomaticsub == False):
 264         opts.writesubtitles = True
 265
 266     if sys.version_info < (3,):
 267         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 268         if opts.outtmpl is not None:
 269             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 270     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 271             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 272             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 273             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 274             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 275             or (opts.useid and u'%(id)s.%(ext)s')
 276             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 277             or DEFAULT_OUTTMPL)
 278     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 279         parser.error(u'Cannot download a video and extract audio into the same'
 280                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 281                      u' template'.format(outtmpl))
 282
 283     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 284     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 285
 286     ydl_opts = {
 287         'usenetrc': opts.usenetrc,
 288         'username': opts.username,
 289         'password': opts.password,
 290         'twofactor': opts.twofactor,
 291         'videopassword': opts.videopassword,
 292         'quiet': (opts.quiet or any_printing),
 293         'no_warnings': opts.no_warnings,
 294         'forceurl': opts.geturl,
 295         'forcetitle': opts.gettitle,
 296         'forceid': opts.getid,
 297         'forcethumbnail': opts.getthumbnail,
 298         'forcedescription': opts.getdescription,
 299         'forceduration': opts.getduration,
 300         'forcefilename': opts.getfilename,
 301         'forceformat': opts.getformat,
 302         'forcejson': opts.dumpjson,
 303         'simulate': opts.simulate,
 304         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 305         'format': opts.format,
 306         'format_limit': opts.format_limit,
 307         'listformats': opts.listformats,
 308         'outtmpl': outtmpl,
 309         'autonumber_size': opts.autonumber_size,
 310         'restrictfilenames': opts.restrictfilenames,
 311         'ignoreerrors': opts.ignoreerrors,
 312         'ratelimit': opts.ratelimit,
 313         'nooverwrites': opts.nooverwrites,
 314         'retries': opts.retries,
 315         'buffersize': opts.buffersize,
 316         'noresizebuffer': opts.noresizebuffer,
 317         'continuedl': opts.continue_dl,
 318         'noprogress': opts.noprogress,
 319         'progress_with_newline': opts.progress_with_newline,
 320         'playliststart': opts.playliststart,
 321         'playlistend': opts.playlistend,
 322         'noplaylist': opts.noplaylist,
 323         'logtostderr': opts.outtmpl == '-',
 324         'consoletitle': opts.consoletitle,
 325         'nopart': opts.nopart,
 326         'updatetime': opts.updatetime,
 327         'writedescription': opts.writedescription,
 328         'writeannotations': opts.writeannotations,
 329         'writeinfojson': opts.writeinfojson,
 330         'writethumbnail': opts.writethumbnail,
 331         'writesubtitles': opts.writesubtitles,
 332         'writeautomaticsub': opts.writeautomaticsub,
 333         'allsubtitles': opts.allsubtitles,
 334         'listsubtitles': opts.listsubtitles,
 335         'subtitlesformat': opts.subtitlesformat,
 336         'subtitleslangs': opts.subtitleslangs,
 337         'matchtitle': decodeOption(opts.matchtitle),
 338         'rejecttitle': decodeOption(opts.rejecttitle),
 339         'max_downloads': opts.max_downloads,
 340         'prefer_free_formats': opts.prefer_free_formats,
 341         'verbose': opts.verbose,
 342         'dump_intermediate_pages': opts.dump_intermediate_pages,
 343         'write_pages': opts.write_pages,
 344         'test': opts.test,
 345         'keepvideo': opts.keepvideo,
 346         'min_filesize': opts.min_filesize,
 347         'max_filesize': opts.max_filesize,
 348         'min_views': opts.min_views,
 349         'max_views': opts.max_views,
 350         'daterange': date,
 351         'cachedir': opts.cachedir,
 352         'youtube_print_sig_code': opts.youtube_print_sig_code,
 353         'age_limit': opts.age_limit,
 354         'download_archive': download_archive_fn,
 355         'cookiefile': opts.cookiefile,
 356         'nocheckcertificate': opts.no_check_certificate,
 357         'prefer_insecure': opts.prefer_insecure,
 358         'proxy': opts.proxy,
 359         'socket_timeout': opts.socket_timeout,
 360         'bidi_workaround': opts.bidi_workaround,
 361         'debug_printtraffic': opts.debug_printtraffic,
 362         'prefer_ffmpeg': opts.prefer_ffmpeg,
 363         'include_ads': opts.include_ads,
 364         'default_search': opts.default_search,
 365         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 366         'encoding': opts.encoding,
 367         'exec_cmd': opts.exec_cmd,
 368     }
 369
 370     with YoutubeDL(ydl_opts) as ydl:
 371         ydl.print_debug_header()
 372         ydl.add_default_info_extractors()
 373
 374         # PostProcessors
 375         # Add the metadata pp first, the other pps will copy it
 376         if opts.addmetadata:
 377             ydl.add_post_processor(FFmpegMetadataPP())
 378         if opts.extractaudio:
 379             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 380         if opts.recodevideo:
 381             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 382         if opts.embedsubtitles:
 383             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 384         if opts.xattrs:
 385             ydl.add_post_processor(XAttrMetadataPP())
 386         if opts.embedthumbnail:
 387             if not opts.addmetadata:
 388                 ydl.add_post_processor(FFmpegAudioFixPP())
 389             ydl.add_post_processor(AtomicParsleyPP())
 390
 391
 392         # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
 393         # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
 394         if opts.exec_cmd:
 395             ydl.add_post_processor(ExecAfterDownloadPP(
 396                 verboseOutput=opts.verbose, exec_cmd=opts.exec_cmd))
 397
 398         # Update version
 399         if opts.update_self:
 400             update_self(ydl.to_screen, opts.verbose)
 401
 402         # Remove cache dir
 403         if opts.rm_cachedir:
 404             ydl.cache.remove()
 405
 406         # Maybe do nothing
 407         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 408             if not (opts.update_self or opts.rm_cachedir):
 409                 parser.error(u'you must provide at least one URL')
 410             else:
 411                 sys.exit()
 412
 413         try:
 414             if opts.load_info_filename is not None:
 415                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 416             else:
 417                 retcode = ydl.download(all_urls)
 418         except MaxDownloadsReached:
 419             ydl.to_screen(u'--max-download limit reached, aborting.')
 420             retcode = 101
 421
 422     sys.exit(retcode)
 423
 424
 425 def main(argv=None):
 426     try:
 427         _real_main(argv)
 428     except DownloadError:
 429         sys.exit(1)
 430     except SameFileError:
 431         sys.exit(u'ERROR: fixed output name but more than one file to download')
 432     except KeyboardInterrupt:
 433         sys.exit(u'\nERROR: Interrupted by user')