youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70     'David Fabijan',
  71     'Sebastian Haas',
  72     'Alexander Kirk',
  73     'Erik Johnson',
  74     'Keith Beckman',
  75     'Ole Ernst',
  76     'Aaron McDaniel (mcd1992)',
  77     'Magnus Kolstad',
  78     'Hari Padmanaban',
  79     'Carlos Ramos',
  80     '5moufl',
  81     'lenaten',
  82     'Xavier Beynon'
  83 )
  84
  85 __license__ = 'Public Domain'
  86
  87 import codecs
  88 import io
  89 import os
  90 import random
  91 import sys
  92
  93
  94 from .options import (
  95     parseOpts,
  96 )
  97 from .utils import (
  98     compat_getpass,
  99     compat_print,
 100     DateRange,
 101     DEFAULT_OUTTMPL,
 102     decodeOption,
 103     DownloadError,
 104     MaxDownloadsReached,
 105     preferredencoding,
 106     read_batch_urls,
 107     SameFileError,
 108     setproctitle,
 109     std_headers,
 110     write_string,
 111 )
 112 from .update import update_self
 113 from .downloader import (
 114     FileDownloader,
 115 )
 116 from .extractor import gen_extractors
 117 from .YoutubeDL import YoutubeDL
 118 from .postprocessor import (
 119     AtomicParsleyPP,
 120     FFmpegAudioFixPP,
 121     FFmpegMetadataPP,
 122     FFmpegVideoConvertor,
 123     FFmpegExtractAudioPP,
 124     FFmpegEmbedSubtitlePP,
 125     XAttrMetadataPP,
 126     ExecAfterDownloadPP,
 127 )
 128
 129
 130 def _real_main(argv=None):
 131     # Compatibility fixes for Windows
 132     if sys.platform == 'win32':
 133         # https://github.com/rg3/youtube-dl/issues/820
 134         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 135
 136     setproctitle(u'youtube-dl')
 137
 138     parser, opts, args = parseOpts(argv)
 139
 140     # Set user agent
 141     if opts.user_agent is not None:
 142         std_headers['User-Agent'] = opts.user_agent
 143
 144     # Set referer
 145     if opts.referer is not None:
 146         std_headers['Referer'] = opts.referer
 147
 148     # Custom HTTP headers
 149     if opts.headers is not None:
 150         for h in opts.headers:
 151             if h.find(':', 1) < 0:
 152                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 153             key, value = h.split(':', 2)
 154             if opts.verbose:
 155                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 156             std_headers[key] = value
 157
 158     # Dump user agent
 159     if opts.dump_user_agent:
 160         compat_print(std_headers['User-Agent'])
 161         sys.exit(0)
 162
 163     # Batch file verification
 164     batch_urls = []
 165     if opts.batchfile is not None:
 166         try:
 167             if opts.batchfile == '-':
 168                 batchfd = sys.stdin
 169             else:
 170                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 171             batch_urls = read_batch_urls(batchfd)
 172             if opts.verbose:
 173                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 174         except IOError:
 175             sys.exit(u'ERROR: batch file could not be read')
 176     all_urls = batch_urls + args
 177     all_urls = [url.strip() for url in all_urls]
 178     _enc = preferredencoding()
 179     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 180
 181     extractors = gen_extractors()
 182
 183     if opts.list_extractors:
 184         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 185             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 186             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 187             for mu in matchedUrls:
 188                 compat_print(u'  ' + mu)
 189         sys.exit(0)
 190     if opts.list_extractor_descriptions:
 191         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 192             if not ie._WORKING:
 193                 continue
 194             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 195             if desc is False:
 196                 continue
 197             if hasattr(ie, 'SEARCH_KEY'):
 198                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 199                 _COUNTS = (u'', u'5', u'10', u'all')
 200                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 201             compat_print(desc)
 202         sys.exit(0)
 203
 204
 205     # Conflicting, missing and erroneous options
 206     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 207         parser.error(u'using .netrc conflicts with giving username/password')
 208     if opts.password is not None and opts.username is None:
 209         parser.error(u'account username missing\n')
 210     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 211         parser.error(u'using output template conflicts with using title, video ID or auto number')
 212     if opts.usetitle and opts.useid:
 213         parser.error(u'using title conflicts with using video ID')
 214     if opts.username is not None and opts.password is None:
 215         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 216     if opts.ratelimit is not None:
 217         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 218         if numeric_limit is None:
 219             parser.error(u'invalid rate limit specified')
 220         opts.ratelimit = numeric_limit
 221     if opts.min_filesize is not None:
 222         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 223         if numeric_limit is None:
 224             parser.error(u'invalid min_filesize specified')
 225         opts.min_filesize = numeric_limit
 226     if opts.max_filesize is not None:
 227         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 228         if numeric_limit is None:
 229             parser.error(u'invalid max_filesize specified')
 230         opts.max_filesize = numeric_limit
 231     if opts.retries is not None:
 232         try:
 233             opts.retries = int(opts.retries)
 234         except (TypeError, ValueError):
 235             parser.error(u'invalid retry count specified')
 236     if opts.buffersize is not None:
 237         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 238         if numeric_buffersize is None:
 239             parser.error(u'invalid buffer size specified')
 240         opts.buffersize = numeric_buffersize
 241     if opts.playliststart <= 0:
 242         raise ValueError(u'Playlist start must be positive')
 243     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 244         raise ValueError(u'Playlist end must be greater than playlist start')
 245     if opts.extractaudio:
 246         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 247             parser.error(u'invalid audio format specified')
 248     if opts.audioquality:
 249         opts.audioquality = opts.audioquality.strip('k').strip('K')
 250         if not opts.audioquality.isdigit():
 251             parser.error(u'invalid audio quality specified')
 252     if opts.recodevideo is not None:
 253         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 254             parser.error(u'invalid video recode format specified')
 255     if opts.date is not None:
 256         date = DateRange.day(opts.date)
 257     else:
 258         date = DateRange(opts.dateafter, opts.datebefore)
 259     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 260         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 261
 262     # Do not download videos when there are audio-only formats
 263     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 264         opts.format = 'bestaudio/best'
 265
 266     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 267     # this was the old behaviour if only --all-sub was given.
 268     if opts.allsubtitles and (opts.writeautomaticsub == False):
 269         opts.writesubtitles = True
 270
 271     if sys.version_info < (3,):
 272         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 273         if opts.outtmpl is not None:
 274             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 275     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 276             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 277             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 278             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 279             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 280             or (opts.useid and u'%(id)s.%(ext)s')
 281             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 282             or DEFAULT_OUTTMPL)
 283     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 284         parser.error(u'Cannot download a video and extract audio into the same'
 285                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 286                      u' template'.format(outtmpl))
 287
 288     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 289     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 290
 291     ydl_opts = {
 292         'usenetrc': opts.usenetrc,
 293         'username': opts.username,
 294         'password': opts.password,
 295         'twofactor': opts.twofactor,
 296         'videopassword': opts.videopassword,
 297         'quiet': (opts.quiet or any_printing),
 298         'no_warnings': opts.no_warnings,
 299         'forceurl': opts.geturl,
 300         'forcetitle': opts.gettitle,
 301         'forceid': opts.getid,
 302         'forcethumbnail': opts.getthumbnail,
 303         'forcedescription': opts.getdescription,
 304         'forceduration': opts.getduration,
 305         'forcefilename': opts.getfilename,
 306         'forceformat': opts.getformat,
 307         'forcejson': opts.dumpjson,
 308         'simulate': opts.simulate,
 309         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 310         'format': opts.format,
 311         'format_limit': opts.format_limit,
 312         'listformats': opts.listformats,
 313         'outtmpl': outtmpl,
 314         'autonumber_size': opts.autonumber_size,
 315         'restrictfilenames': opts.restrictfilenames,
 316         'ignoreerrors': opts.ignoreerrors,
 317         'ratelimit': opts.ratelimit,
 318         'nooverwrites': opts.nooverwrites,
 319         'retries': opts.retries,
 320         'buffersize': opts.buffersize,
 321         'noresizebuffer': opts.noresizebuffer,
 322         'continuedl': opts.continue_dl,
 323         'noprogress': opts.noprogress,
 324         'progress_with_newline': opts.progress_with_newline,
 325         'playliststart': opts.playliststart,
 326         'playlistend': opts.playlistend,
 327         'noplaylist': opts.noplaylist,
 328         'logtostderr': opts.outtmpl == '-',
 329         'consoletitle': opts.consoletitle,
 330         'nopart': opts.nopart,
 331         'updatetime': opts.updatetime,
 332         'writedescription': opts.writedescription,
 333         'writeannotations': opts.writeannotations,
 334         'writeinfojson': opts.writeinfojson,
 335         'writethumbnail': opts.writethumbnail,
 336         'writesubtitles': opts.writesubtitles,
 337         'writeautomaticsub': opts.writeautomaticsub,
 338         'allsubtitles': opts.allsubtitles,
 339         'listsubtitles': opts.listsubtitles,
 340         'subtitlesformat': opts.subtitlesformat,
 341         'subtitleslangs': opts.subtitleslangs,
 342         'matchtitle': decodeOption(opts.matchtitle),
 343         'rejecttitle': decodeOption(opts.rejecttitle),
 344         'max_downloads': opts.max_downloads,
 345         'prefer_free_formats': opts.prefer_free_formats,
 346         'verbose': opts.verbose,
 347         'dump_intermediate_pages': opts.dump_intermediate_pages,
 348         'write_pages': opts.write_pages,
 349         'test': opts.test,
 350         'keepvideo': opts.keepvideo,
 351         'min_filesize': opts.min_filesize,
 352         'max_filesize': opts.max_filesize,
 353         'min_views': opts.min_views,
 354         'max_views': opts.max_views,
 355         'daterange': date,
 356         'cachedir': opts.cachedir,
 357         'youtube_print_sig_code': opts.youtube_print_sig_code,
 358         'age_limit': opts.age_limit,
 359         'download_archive': download_archive_fn,
 360         'cookiefile': opts.cookiefile,
 361         'nocheckcertificate': opts.no_check_certificate,
 362         'prefer_insecure': opts.prefer_insecure,
 363         'proxy': opts.proxy,
 364         'socket_timeout': opts.socket_timeout,
 365         'bidi_workaround': opts.bidi_workaround,
 366         'debug_printtraffic': opts.debug_printtraffic,
 367         'prefer_ffmpeg': opts.prefer_ffmpeg,
 368         'include_ads': opts.include_ads,
 369         'default_search': opts.default_search,
 370         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 371         'encoding': opts.encoding,
 372         'exec_cmd': opts.exec_cmd,
 373     }
 374
 375     with YoutubeDL(ydl_opts) as ydl:
 376         ydl.print_debug_header()
 377         ydl.add_default_info_extractors()
 378
 379         # PostProcessors
 380         # Add the metadata pp first, the other pps will copy it
 381         if opts.addmetadata:
 382             ydl.add_post_processor(FFmpegMetadataPP())
 383         if opts.extractaudio:
 384             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 385         if opts.recodevideo:
 386             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 387         if opts.embedsubtitles:
 388             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 389         if opts.xattrs:
 390             ydl.add_post_processor(XAttrMetadataPP())
 391         if opts.embedthumbnail:
 392             if not opts.addmetadata:
 393                 ydl.add_post_processor(FFmpegAudioFixPP())
 394             ydl.add_post_processor(AtomicParsleyPP())
 395
 396
 397         # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
 398         # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
 399         if opts.exec_cmd:
 400             ydl.add_post_processor(ExecAfterDownloadPP(
 401                 verboseOutput=opts.verbose, exec_cmd=opts.exec_cmd))
 402
 403         # Update version
 404         if opts.update_self:
 405             update_self(ydl.to_screen, opts.verbose)
 406
 407         # Remove cache dir
 408         if opts.rm_cachedir:
 409             ydl.cache.remove()
 410
 411         # Maybe do nothing
 412         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 413             if not (opts.update_self or opts.rm_cachedir):
 414                 parser.error(u'you must provide at least one URL')
 415             else:
 416                 sys.exit()
 417
 418         try:
 419             if opts.load_info_filename is not None:
 420                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 421             else:
 422                 retcode = ydl.download(all_urls)
 423         except MaxDownloadsReached:
 424             ydl.to_screen(u'--max-download limit reached, aborting.')
 425             retcode = 101
 426
 427     sys.exit(retcode)
 428
 429
 430 def main(argv=None):
 431     try:
 432         _real_main(argv)
 433     except DownloadError:
 434         sys.exit(1)
 435     except SameFileError:
 436         sys.exit(u'ERROR: fixed output name but more than one file to download')
 437     except KeyboardInterrupt:
 438         sys.exit(u'\nERROR: Interrupted by user')