import re
import shutil
import subprocess
-import socket
import sys
import time
import tokenize
compat_basestring,
compat_cookiejar,
compat_get_terminal_size,
- compat_http_client,
compat_kwargs,
compat_numeric_types,
compat_os_name,
date_from_str,
DateRange,
DEFAULT_OUTTMPL,
- OUTTMPL_TYPES,
determine_ext,
determine_protocol,
DOT_DESKTOP_LINK_TEMPLATE,
DownloadError,
encode_compat_str,
encodeFilename,
- error_to_compat_str,
EntryNotInPlaylist,
+ error_to_compat_str,
ExistingVideoReached,
expand_path,
ExtractorError,
make_dir,
make_HTTPS_handler,
MaxDownloadsReached,
+ network_exceptions,
orderedSet,
+ OUTTMPL_TYPES,
PagedList,
parse_filesize,
PerRequestProxyHandler,
PostProcessingError,
preferredencoding,
prepend_extension,
+ process_communicate_or_kill,
+ random_uuidv4,
register_socks_protocols,
+ RejectedVideoReached,
render_table,
replace_extension,
- RejectedVideoReached,
SameFileError,
sanitize_filename,
sanitize_path,
strftime_or_none,
subtitles_filename,
to_high_limit_path,
+ traverse_dict,
UnavailableVideoError,
url_basename,
version_tuple,
YoutubeDLCookieProcessor,
YoutubeDLHandler,
YoutubeDLRedirectHandler,
- process_communicate_or_kill,
)
from .cache import Cache
-from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER, _PLUGIN_CLASSES
+from .extractor import (
+ gen_extractor_classes,
+ get_info_extractor,
+ _LAZY_LOADER,
+ _PLUGIN_CLASSES
+)
from .extractor.openload import PhantomJSwrapper
-from .downloader import get_suitable_downloader
+from .downloader import (
+ get_suitable_downloader,
+ shorten_protocol_name
+)
from .downloader.rtmp import rtmpdump_version
from .postprocessor import (
FFmpegFixupM3u8PP,
verbose: Print additional info to stdout.
quiet: Do not print messages to stdout.
no_warnings: Do not print out anything for warnings.
- forceurl: Force printing final URL.
- forcetitle: Force printing title.
- forceid: Force printing ID.
- forcethumbnail: Force printing thumbnail URL.
- forcedescription: Force printing description.
- forcefilename: Force printing final filename.
- forceduration: Force printing duration.
+ forceprint: A list of templates to force print
+ forceurl: Force printing final URL. (Deprecated)
+ forcetitle: Force printing title. (Deprecated)
+ forceid: Force printing ID. (Deprecated)
+ forcethumbnail: Force printing thumbnail URL. (Deprecated)
+ forcedescription: Force printing description. (Deprecated)
+ forcefilename: Force printing final filename. (Deprecated)
+ forceduration: Force printing duration. (Deprecated)
forcejson: Force printing info_dict as JSON.
dump_single_json: Force printing the info_dict of the whole playlist
(or video) as a single JSON line.
simulate: Do not download the video files.
format: Video format code. see "FORMAT SELECTION" for more details.
allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
+ ignore_no_formats_error: Ignore "No video formats" error. Usefull for
+ extracting metadata even if the video is not actually
+ available for download (experimental)
format_sort: How to sort the video formats. see "Sorting Formats"
for more details.
format_sort_force: Force the given format_sort. see "Sorting Formats"
ignoreerrors: Do not stop on download errors
(Default True when running yt-dlp,
but False when directly accessing YoutubeDL class)
+ skip_playlist_after_errors: Number of allowed failures until the rest of
+ the playlist is skipped
force_generic_extractor: Force downloader to use the generic extractor
overwrites: Overwrite all video and metadata files if True,
overwrite only non-video files if None
writedesktoplink: Write a Linux internet shortcut file (.desktop)
writesubtitles: Write the video subtitles to a file
writeautomaticsub: Write the automatically generated subtitles to a file
- allsubtitles: Downloads all the subtitles of the video
+ allsubtitles: Deprecated - Use subtitlelangs = ['all']
+ Downloads all the subtitles of the video
(requires writesubtitles or writeautomaticsub)
listsubtitles: Lists all available subtitles for the video
subtitlesformat: The format code for subtitles
- subtitleslangs: List of languages of the subtitles to download
+ subtitleslangs: List of languages of the subtitles to download (can be regex).
+ The list may contain "all" to refer to all the available
+ subtitles. The language can be prefixed with a "-" to
+ exclude it from the requested languages. Eg: ['all', '-live_chat']
keepvideo: Keep the video file after post-processing
daterange: A DateRange object, download only if the upload_date is in the range.
skip_download: Skip the actual download of the video file
postprocessors: A list of dictionaries, each with an entry
* key: The name of the postprocessor. See
yt_dlp/postprocessor/__init__.py for a list.
- * _after_move: Optional. If True, run this post_processor
- after 'MoveFilesAfterDownload'
- as well as any further keyword arguments for the
- postprocessor.
+ * when: When to run the postprocessor. Can be one of
+ pre_process|before_dl|post_process|after_move.
+ Assumed to be 'post_process' if not given
post_hooks: A list of functions that get called as the final step
for each video file, after all postprocessors have been
called. The filename will be passed as the only argument.
geo_bypass_country
The following options determine which downloader is picked:
- external_downloader: Executable of the external downloader to call.
- None or unset for standard (built-in) downloader.
- hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
+ external_downloader: A dictionary of protocol keys and the executable of the
+ external downloader to use for it. The allowed protocols
+ are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
+ Set the value to 'native' to use the native downloader
+ hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
+ or {'m3u8': 'ffmpeg'} instead.
+ Use the native HLS downloader instead of ffmpeg/avconv
if True, otherwise use ffmpeg/avconv if False, otherwise
use downloader suggested by extractor if None.
+ compat_opts: Compatibility options. See "Differences in default behavior".
+ Note that only format-sort, format-spec, no-live-chat, no-attach-info-json
+ playlist-index, list-formats, no-youtube-channel-redirect
+ and no-youtube-unavailable-videos works when used via the API
The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see yt_dlp/downloader/common.py):
params = None
_ies = []
- _pps = {'beforedl': [], 'aftermove': [], 'normal': []}
+ _pps = {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
__prepare_filename_warned = False
_first_webpage_request = True
_download_retcode = None
params = {}
self._ies = []
self._ies_instances = {}
- self._pps = {'beforedl': [], 'aftermove': [], 'normal': []}
+ self._pps = {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
self.__prepare_filename_warned = False
self._first_webpage_request = True
self._post_hooks = []
}
self.params.update(params)
self.cache = Cache(self)
- self.archive = set()
- """Preload the archive, if any is specified"""
- def preload_download_archive(self):
- fn = self.params.get('download_archive')
- if fn is None:
- return False
- try:
- with locked_file(fn, 'r', encoding='utf-8') as archive_file:
- for line in archive_file:
- self.archive.add(line.strip())
- except IOError as ioe:
- if ioe.errno != errno.ENOENT:
- raise
- return False
- return True
+ if sys.version_info < (3, 6):
+ self.report_warning(
+ 'Support for Python version %d.%d have been deprecated and will break in future versions of yt-dlp! '
+ 'Update to Python 3.6 or above' % sys.version_info[:2])
def check_deprecated(param, option, suggestion):
if self.params.get(param) is not None:
- self.report_warning(
- '%s is deprecated. Use %s instead.' % (option, suggestion))
+ self.report_warning('%s is deprecated. Use %s instead' % (option, suggestion))
return True
return False
- if self.params.get('verbose'):
- self.to_stdout('[debug] Loading archive file %r' % self.params.get('download_archive'))
-
- preload_download_archive(self)
-
if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
if self.params.get('geo_verification_proxy') is None:
self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
+ check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
+ check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
+ check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
+
+ for msg in self.params.get('warnings', []):
+ self.report_warning(msg)
+
if self.params.get('final_ext'):
if self.params.get('merge_output_format'):
self.report_warning('--merge-output-format will be ignored since --remux-video or --recode-video is given')
if 'overwrites' in self.params and self.params['overwrites'] is None:
del self.params['overwrites']
- check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
- check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
- check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
-
if params.get('bidi_workaround', False):
try:
import pty
self._setup_opener()
+ """Preload the archive, if any is specified"""
+ def preload_download_archive(fn):
+ if fn is None:
+ return False
+ self.write_debug('Loading archive file %r\n' % fn)
+ try:
+ with locked_file(fn, 'r', encoding='utf-8') as archive_file:
+ for line in archive_file:
+ self.archive.add(line.strip())
+ except IOError as ioe:
+ if ioe.errno != errno.ENOENT:
+ raise
+ return False
+ return True
+
+ self.archive = set()
+ preload_download_archive(self.params.get('download_archive'))
+
if auto_init:
self.print_debug_header()
self.add_default_info_extractors()
when = pp_def['when']
del pp_def['when']
else:
- when = 'normal'
+ when = 'post_process'
pp = pp_class(self, **compat_kwargs(pp_def))
self.add_post_processor(pp, when=when)
for ie in gen_extractor_classes():
self.add_info_extractor(ie)
- def add_post_processor(self, pp, when='normal'):
+ def add_post_processor(self, pp, when='post_process'):
"""Add a PostProcessor object to the end of the chain."""
self._pps[when].append(pp)
pp.set_downloader(self)
for _ in range(line_count))
return res[:-len('\n')]
- def to_screen(self, message, skip_eol=False):
- """Print message to stdout if not in quiet mode."""
- return self.to_stdout(message, skip_eol, check_quiet=True)
-
def _write_string(self, s, out=None):
write_string(s, out=out, encoding=self.params.get('encoding'))
- def to_stdout(self, message, skip_eol=False, check_quiet=False):
- """Print message to stdout if not in quiet mode."""
+ def to_stdout(self, message, skip_eol=False, quiet=False):
+ """Print message to stdout"""
if self.params.get('logger'):
self.params['logger'].debug(message)
- elif not check_quiet or not self.params.get('quiet', False):
+ elif not quiet:
message = self._bidi_workaround(message)
terminator = ['\n', ''][skip_eol]
output = message + terminator
self._write_string(output, self._screen_file)
def to_stderr(self, message):
- """Print message to stderr."""
+ """Print message to stderr"""
assert isinstance(message, compat_str)
if self.params.get('logger'):
self.params['logger'].error(message)
raise DownloadError(message, exc_info)
self._download_retcode = 1
+ def to_screen(self, message, skip_eol=False):
+ """Print message to stdout if not in quiet mode"""
+ self.to_stdout(
+ message, skip_eol, quiet=self.params.get('quiet', False))
+
def report_warning(self, message):
'''
Print the message to stderr, it will be prefixed with 'WARNING:'
error_message = '%s %s' % (_msg_header, message)
self.trouble(error_message, tb)
+ def write_debug(self, message):
+ '''Log debug message or Print message to stderr'''
+ if not self.params.get('verbose', False):
+ return
+ message = '[debug] %s' % message
+ if self.params.get('logger'):
+ self.params['logger'].debug(message)
+ else:
+ self._write_string('%s\n' % message)
+
def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded."""
try:
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
""" Make the template and info_dict suitable for substitution (outtmpl % info_dict)"""
template_dict = dict(info_dict)
+ na = self.params.get('outtmpl_na_placeholder', 'NA')
# duration_string
template_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
- formatSeconds(info_dict['duration'], '-')
+ formatSeconds(info_dict['duration'], '-' if sanitize else ':')
if info_dict.get('duration', None) is not None
else None)
elif template_dict.get('width'):
template_dict['resolution'] = '%dx?' % template_dict['width']
- if sanitize is None:
- sanitize = lambda k, v: v
- template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
- for k, v in template_dict.items()
- if v is not None and not isinstance(v, (list, tuple, dict)))
- na = self.params.get('outtmpl_na_placeholder', 'NA')
- template_dict = collections.defaultdict(lambda: na, template_dict)
-
# For fields playlist_index and autonumber convert all occurrences
# of %(field)s to %(field)0Nd for backward compatibility
field_size_compat_map = {
- 'playlist_index': len(str(template_dict['n_entries'])),
+ 'playlist_index': len(str(template_dict.get('_last_playlist_index') or '')),
'autonumber': autonumber_size,
}
FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
outtmpl)
numeric_fields = list(self._NUMERIC_FIELDS)
+ if sanitize is None:
+ sanitize = lambda k, v: v
- # Format date
- FORMAT_DATE_RE = FORMAT_RE.format(r'(?P<key>(?P<field>\w+)>(?P<format>.+?))')
- for mobj in re.finditer(FORMAT_DATE_RE, outtmpl):
- conv_type, field, frmt, key = mobj.group('type', 'field', 'format', 'key')
- if key in template_dict:
- continue
- value = strftime_or_none(template_dict.get(field), frmt, na)
- if conv_type in 'crs': # string
- value = sanitize(field, value)
- else: # number
- numeric_fields.append(key)
- value = float_or_none(value, default=None)
+ EXTERNAL_FORMAT_RE = FORMAT_RE.format('(?P<key>[^)]*)')
+ # Field is of the form key1.key2...
+ # where keys (except first) can be string, int or slice
+ FIELD_RE = r'\w+(?:\.(?:\w+|[-\d]*(?::[-\d]*){0,2}))*'
+ INTERNAL_FORMAT_RE = re.compile(r'''(?x)
+ (?P<negate>-)?
+ (?P<fields>{0})
+ (?P<maths>(?:[-+]-?(?:\d+(?:\.\d+)?|{0}))*)
+ (?:>(?P<strf_format>.+?))?
+ (?:\|(?P<default>.*?))?
+ $'''.format(FIELD_RE))
+ MATH_OPERATORS_RE = re.compile(r'(?<![-+])([-+])')
+ MATH_FUNCTIONS = {
+ '+': float.__add__,
+ '-': float.__sub__,
+ }
+ for outer_mobj in re.finditer(EXTERNAL_FORMAT_RE, outtmpl):
+ final_key = outer_mobj.group('key')
+ str_type = outer_mobj.group('type')
+ value = None
+ mobj = re.match(INTERNAL_FORMAT_RE, final_key)
+ if mobj is not None:
+ mobj = mobj.groupdict()
+ # Object traversal
+ fields = mobj['fields'].split('.')
+ value = traverse_dict(template_dict, fields)
+ # Negative
+ if mobj['negate']:
+ value = float_or_none(value)
+ if value is not None:
+ value *= -1
+ # Do maths
+ if mobj['maths']:
+ value = float_or_none(value)
+ operator = None
+ for item in MATH_OPERATORS_RE.split(mobj['maths'])[1:]:
+ if item == '':
+ value = None
+ if value is None:
+ break
+ if operator:
+ item, multiplier = (item[1:], -1) if item[0] == '-' else (item, 1)
+ offset = float_or_none(item)
+ if offset is None:
+ offset = float_or_none(traverse_dict(template_dict, item.split('.')))
+ try:
+ value = operator(value, multiplier * offset)
+ except (TypeError, ZeroDivisionError):
+ value = None
+ operator = None
+ else:
+ operator = MATH_FUNCTIONS[item]
+ # Datetime formatting
+ if mobj['strf_format']:
+ value = strftime_or_none(value, mobj['strf_format'])
+ # Set default
+ if value is None and mobj['default'] is not None:
+ value = mobj['default']
+ # Sanitize
+ if str_type in 'crs' and value is not None: # string
+ value = sanitize('%{}'.format(str_type) % fields[-1], value)
+ else: # numeric
+ numeric_fields.append(final_key)
+ value = float_or_none(value)
if value is not None:
- template_dict[key] = value
+ template_dict[final_key] = value
# Missing numeric fields used together with integer presentation types
# in format specification will break the argument substitution since
# string NA placeholder is returned for missing fields. We will patch
# output template for missing fields to meet string presentation type.
for numeric_field in numeric_fields:
- if numeric_field not in template_dict:
+ if template_dict.get(numeric_field) is None:
outtmpl = re.sub(
FORMAT_RE.format(re.escape(numeric_field)),
r'%({0})s'.format(numeric_field), outtmpl)
+ template_dict = collections.defaultdict(lambda: na, (
+ (k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
+ for k, v in template_dict.items() if v is not None))
return outtmpl, template_dict
def _prepare_filename(self, info_dict, tmpl_type='default'):
for key, value in extra_info.items():
info_dict.setdefault(key, value)
- def extract_info(self, url, download=True, ie_key=None, info_dict=None, extra_info={},
+ def extract_info(self, url, download=True, ie_key=None, extra_info={},
process=True, force_generic_extractor=False):
- '''
- Returns a list with a dictionary for each video we find.
- If 'download', also downloads the videos.
- extra_info is a dict containing the extra values to add to each result
- '''
+ """
+ Return a list with a dictionary for each video extracted.
+
+ Arguments:
+ url -- URL to extract
+
+ Keyword arguments:
+ download -- whether to download videos during extraction
+ ie_key -- extractor key hint
+ extra_info -- dictionary containing the extra values to add to each result
+ process -- whether to resolve all unresolved references (URLs, playlist items),
+ must be True for download to work.
+ force_generic_extractor -- force using the generic extractor
+ """
if not ie_key and force_generic_extractor:
ie_key = 'Generic'
self.to_screen("[%s] %s: has already been recorded in archive" % (
ie_key, temp_id))
break
- return self.__extract_info(url, ie, download, extra_info, process, info_dict)
+ return self.__extract_info(url, ie, download, extra_info, process)
else:
self.report_error('no suitable InfoExtractor for URL %s' % url)
return wrapper
@__handle_extraction_exceptions
- def __extract_info(self, url, ie, download, extra_info, process, info_dict):
+ def __extract_info(self, url, ie, download, extra_info, process):
ie_result = ie.extract(url)
if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
return
'_type': 'compat_list',
'entries': ie_result,
}
- if info_dict:
- if info_dict.get('id'):
- ie_result['id'] = info_dict['id']
- if info_dict.get('title'):
- ie_result['title'] = info_dict['title']
self.add_default_extra_info(ie_result, ie, url)
if process:
return self.process_ie_result(ie_result, download, extra_info)
if result_type == 'video':
self.add_extra_info(ie_result, extra_info)
- return self.process_video_result(ie_result, download=download)
+ ie_result = self.process_video_result(ie_result, download=download)
+ additional_urls = ie_result.get('additional_urls')
+ if additional_urls:
+ # TODO: Improve MetadataFromFieldPP to allow setting a list
+ if isinstance(additional_urls, compat_str):
+ additional_urls = [additional_urls]
+ self.to_screen(
+ '[info] %s: %d additional URL(s) requested' % (ie_result['id'], len(additional_urls)))
+ self.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls))
+ ie_result['additional_entries'] = [
+ self.extract_info(
+ url, download, extra_info,
+ force_generic_extractor=self.params.get('force_generic_extractor'))
+ for url in additional_urls
+ ]
+ return ie_result
elif result_type == 'url':
# We have to add extra_info to the results because it may be
# contained in a playlist
- return self.extract_info(ie_result['url'],
- download, info_dict=ie_result,
- ie_key=ie_result.get('ie_key'),
- extra_info=extra_info)
+ return self.extract_info(
+ ie_result['url'], download,
+ ie_key=ie_result.get('ie_key'),
+ extra_info=extra_info)
elif result_type == 'url_transparent':
# Use the information from the embedding page
info = self.extract_info(
'playlist_title': ie_result.get('title'),
'playlist_uploader': ie_result.get('uploader'),
'playlist_uploader_id': ie_result.get('uploader_id'),
- 'playlist_index': 0
+ 'playlist_index': 0,
}
ie_copy.update(dict(ie_result))
except (OSError, IOError):
self.report_error('Cannot write playlist metadata to JSON file ' + infofn)
+ # TODO: This should be passed to ThumbnailsConvertor if necessary
+ self._write_thumbnails(ie_copy, self.prepare_filename(ie_copy, 'pl_thumbnail'))
+
if self.params.get('writedescription', False):
descfn = self.prepare_filename(ie_copy, 'pl_description')
if not self._ensure_dir_exists(encodeFilename(descfn)):
self.report_error('Cannot write playlist description file ' + descfn)
return
+ # Save playlist_index before re-ordering
+ entries = [
+ ((playlistitems[i - 1] if playlistitems else i), entry)
+ for i, entry in enumerate(entries, 1)]
+
if self.params.get('playlistreverse', False):
entries = entries[::-1]
if self.params.get('playlistrandom', False):
x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
self.to_screen('[%s] playlist %s: %s' % (ie_result['extractor'], playlist, msg))
- for i, entry in enumerate(entries, 1):
+ failures = 0
+ max_failures = self.params.get('skip_playlist_after_errors') or float('inf')
+ for i, entry_tuple in enumerate(entries, 1):
+ playlist_index, entry = entry_tuple
+ if 'playlist_index' in self.params.get('compat_options', []):
+ playlist_index = playlistitems[i - 1] if playlistitems else i
self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
# This __x_forwarded_for_ip thing is a bit ugly but requires
# minimal changes
entry['__x_forwarded_for_ip'] = x_forwarded_for
extra = {
'n_entries': n_entries,
+ '_last_playlist_index': max(playlistitems) if playlistitems else (playlistend or n_entries),
+ 'playlist_index': playlist_index,
+ 'playlist_autonumber': i,
'playlist': playlist,
'playlist_id': ie_result.get('id'),
'playlist_title': ie_result.get('title'),
'playlist_uploader': ie_result.get('uploader'),
'playlist_uploader_id': ie_result.get('uploader_id'),
- 'playlist_index': playlistitems[i - 1] if playlistitems else i,
'extractor': ie_result['extractor'],
'webpage_url': ie_result['webpage_url'],
'webpage_url_basename': url_basename(ie_result['webpage_url']),
continue
entry_result = self.__process_iterable_entry(entry, download, extra)
+ if not entry_result:
+ failures += 1
+ if failures >= max_failures:
+ self.report_error(
+ 'Skipping the remaining entries in playlist "%s" since %d items failed extraction' % (playlist, failures))
+ break
# TODO: skip failed (empty) entries?
playlist_results.append(entry_result)
ie_result['entries'] = playlist_results
not can_merge()
or info_dict.get('is_live', False)
or self.outtmpl_dict['default'] == '-'))
+ compat = (
+ prefer_best
+ or self.params.get('allow_multiple_audio_streams', False)
+ or 'format-spec' in self.params.get('compat_opts', []))
return (
- 'best/bestvideo+bestaudio'
- if prefer_best
- else 'bestvideo*+bestaudio/best'
- if not self.params.get('allow_multiple_audio_streams', False)
+ 'best/bestvideo+bestaudio' if prefer_best
+ else 'bestvideo*+bestaudio/best' if not compat
else 'bestvideo+bestaudio/best')
def build_format_selector(self, format_spec):
allow_multiple_streams = {'audio': self.params.get('allow_multiple_audio_streams', False),
'video': self.params.get('allow_multiple_video_streams', False)}
+ check_formats = self.params.get('check_formats')
+
def _parse_filter(tokens):
filter_parts = []
for type, string, start, _, _ in tokens:
return new_dict
+ def _check_formats(formats):
+ for f in formats:
+ self.to_screen('[info] Testing format %s' % f['format_id'])
+ paths = self.params.get('paths', {})
+ temp_file = os.path.join(
+ expand_path(paths.get('home', '').strip()),
+ expand_path(paths.get('temp', '').strip()),
+ 'ytdl.%s.f%s.check-format' % (random_uuidv4(), f['format_id']))
+ try:
+ dl, _ = self.dl(temp_file, f, test=True)
+ except (ExtractorError, IOError, OSError, ValueError) + network_exceptions:
+ dl = False
+ finally:
+ if os.path.exists(temp_file):
+ os.remove(temp_file)
+ if dl:
+ yield f
+ else:
+ self.to_screen('[info] Unable to download format %s. Skipping...' % f['format_id'])
+
def _build_selector_function(selector):
if isinstance(selector, list): # ,
fs = [_build_selector_function(s) for s in selector]
return []
elif selector.type == SINGLE: # atom
- format_spec = (selector.selector if selector.selector is not None else 'best').lower()
+ format_spec = selector.selector or 'best'
# TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
if format_spec == 'all':
def selector_function(ctx):
formats = list(ctx['formats'])
- if formats:
- for f in formats:
- yield f
+ if check_formats:
+ formats = _check_formats(formats)
+ for f in formats:
+ yield f
elif format_spec == 'mergeall':
def selector_function(ctx):
- formats = list(ctx['formats'])
- merged_format = formats[0]
- for f in formats[1:]:
+ formats = list(_check_formats(ctx['formats']))
+ if not formats:
+ return
+ merged_format = formats[-1]
+ for f in formats[-2::-1]:
merged_format = _merge((merged_format, f))
yield merged_format
else:
- format_fallback = False
+ format_fallback, format_reverse, format_idx = False, True, 1
mobj = re.match(
r'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
format_spec)
if mobj is not None:
format_idx = int_or_none(mobj.group('n'), default=1)
- format_idx = format_idx - 1 if mobj.group('bw')[0] == 'w' else -format_idx
+ format_reverse = mobj.group('bw')[0] == 'b'
format_type = (mobj.group('type') or [None])[0]
not_format_type = {'v': 'a', 'a': 'v'}.get(format_type)
format_modified = mobj.group('mod') is not None
if not format_modified # b, w
else None) # b*, w*
else:
- format_idx = -1
filter_f = ((lambda f: f.get('ext') == format_spec)
if format_spec in ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav'] # extension
else (lambda f: f.get('format_id') == format_spec)) # id
if not formats:
return
matches = list(filter(filter_f, formats)) if filter_f is not None else formats
- n = len(matches)
- if -n <= format_idx < n:
- yield matches[format_idx]
- elif format_fallback and ctx['incomplete_formats']:
+ if format_fallback and ctx['incomplete_formats'] and not matches:
# for extractors with incomplete formats (audio only (soundcloud)
# or video only (imgur)) best/worst will fallback to
# best/worst {video,audio}-only format
- n = len(formats)
- if -n <= format_idx < n:
- yield formats[format_idx]
+ matches = formats
+ if format_reverse:
+ matches = matches[::-1]
+ if check_formats:
+ matches = list(itertools.islice(_check_formats(matches), format_idx))
+ n = len(matches)
+ if -n <= format_idx - 1 < n:
+ yield matches[format_idx - 1]
elif selector.type == MERGE: # +
selector_1, selector_2 = map(_build_selector_function, selector.selector)
t.get('preference') if t.get('preference') is not None else -1,
t.get('width') if t.get('width') is not None else -1,
t.get('height') if t.get('height') is not None else -1,
- t.get('id') if t.get('id') is not None else '', t.get('url')))
+ t.get('id') if t.get('id') is not None else '',
+ t.get('url')))
for i, t in enumerate(thumbnails):
t['url'] = sanitize_url(t['url'])
if t.get('width') and t.get('height'):
formats = info_dict['formats']
if not formats:
- raise ExtractorError('No video formats found!')
+ if not self.params.get('ignore_no_formats_error'):
+ raise ExtractorError('No video formats found!')
+ else:
+ self.report_warning('No video formats found!')
def is_wellformed(f):
url = f.get('url')
# TODO Central sorting goes here
- if formats[0] is not info_dict:
+ if formats and formats[0] is not info_dict:
# only set the 'formats' fields if the original info_dict list them
# otherwise we end up with a circular reference, the first (and unique)
# element in the 'formats' field in info_dict is info_dict itself,
# which can't be exported to json
info_dict['formats'] = formats
+
+ info_dict, _ = self.pre_process(info_dict)
+
if self.params.get('listformats'):
+ if not info_dict.get('formats'):
+ raise ExtractorError('No video formats found', expected=True)
self.list_formats(info_dict)
return
req_format = self.params.get('format')
if req_format is None:
req_format = self._default_format_spec(info_dict, download=download)
- if self.params.get('verbose'):
- self.to_screen('[debug] Default format spec: %s' % req_format)
+ self.write_debug('Default format spec: %s' % req_format)
format_selector = self.build_format_selector(req_format)
formats_to_download = list(format_selector(ctx))
if not formats_to_download:
- raise ExtractorError('requested format not available',
- expected=True)
-
- if download:
- self.to_screen('[info] Downloading format(s) %s' % ", ".join([f['format_id'] for f in formats_to_download]))
- if len(formats_to_download) > 1:
- self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
- for format in formats_to_download:
+ if not self.params.get('ignore_no_formats_error'):
+ raise ExtractorError('Requested format is not available', expected=True)
+ else:
+ self.report_warning('Requested format is not available')
+ elif download:
+ self.to_screen(
+ '[info] %s: Downloading %d format(s): %s' % (
+ info_dict['id'], len(formats_to_download),
+ ", ".join([f['format_id'] for f in formats_to_download])))
+ for fmt in formats_to_download:
new_info = dict(info_dict)
- new_info.update(format)
+ # Save a reference to the original info_dict so that it can be modified in process_info if needed
+ new_info['__original_infodict'] = info_dict
+ new_info.update(fmt)
self.process_info(new_info)
# We update the info dict with the best quality format (backwards compatibility)
- info_dict.update(formats_to_download[-1])
+ if formats_to_download:
+ info_dict.update(formats_to_download[-1])
return info_dict
def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
available_subs):
return None
+ all_sub_langs = available_subs.keys()
if self.params.get('allsubtitles', False):
- requested_langs = available_subs.keys()
+ requested_langs = all_sub_langs
+ elif self.params.get('subtitleslangs', False):
+ requested_langs = set()
+ for lang in self.params.get('subtitleslangs'):
+ if lang == 'all':
+ requested_langs.update(all_sub_langs)
+ continue
+ discard = lang[0] == '-'
+ if discard:
+ lang = lang[1:]
+ current_langs = filter(re.compile(lang + '$').match, all_sub_langs)
+ if discard:
+ for lang in current_langs:
+ requested_langs.discard(lang)
+ else:
+ requested_langs.update(current_langs)
+ elif 'en' in available_subs:
+ requested_langs = ['en']
else:
- if self.params.get('subtitleslangs', False):
- requested_langs = self.params.get('subtitleslangs')
- elif 'en' in available_subs:
- requested_langs = ['en']
- else:
- requested_langs = [list(available_subs.keys())[0]]
+ requested_langs = [list(all_sub_langs)[0]]
+ self.write_debug('Downloading subtitles: %s' % ', '.join(requested_langs))
formats_query = self.params.get('subtitlesformat', 'best')
formats_preference = formats_query.split('/') if formats_query else []
return subs
def __forced_printings(self, info_dict, filename, incomplete):
- def print_mandatory(field):
+ def print_mandatory(field, actual_field=None):
+ if actual_field is None:
+ actual_field = field
if (self.params.get('force%s' % field, False)
- and (not incomplete or info_dict.get(field) is not None)):
- self.to_stdout(info_dict[field])
+ and (not incomplete or info_dict.get(actual_field) is not None)):
+ self.to_stdout(info_dict[actual_field])
def print_optional(field):
if (self.params.get('force%s' % field, False)
and info_dict.get(field) is not None):
self.to_stdout(info_dict[field])
+ info_dict = info_dict.copy()
+ if filename is not None:
+ info_dict['filename'] = filename
+ if info_dict.get('requested_formats') is not None:
+ # For RTMP URLs, also include the playpath
+ info_dict['urls'] = '\n'.join(f['url'] + f.get('play_path', '') for f in info_dict['requested_formats'])
+ elif 'url' in info_dict:
+ info_dict['urls'] = info_dict['url'] + info_dict.get('play_path', '')
+
+ for tmpl in self.params.get('forceprint', []):
+ if re.match(r'\w+$', tmpl):
+ tmpl = '%({})s'.format(tmpl)
+ tmpl, info_copy = self.prepare_outtmpl(tmpl, info_dict)
+ self.to_stdout(tmpl % info_copy)
+
print_mandatory('title')
print_mandatory('id')
- if self.params.get('forceurl', False) and not incomplete:
- if info_dict.get('requested_formats') is not None:
- for f in info_dict['requested_formats']:
- self.to_stdout(f['url'] + f.get('play_path', ''))
- else:
- # For RTMP URLs, also include the playpath
- self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
+ print_mandatory('url', 'urls')
print_optional('thumbnail')
print_optional('description')
- if self.params.get('forcefilename', False) and filename is not None:
- self.to_stdout(filename)
+ print_optional('filename')
if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
self.to_stdout(formatSeconds(info_dict['duration']))
print_mandatory('format')
+
if self.params.get('forcejson', False):
self.post_extract(info_dict)
self.to_stdout(json.dumps(info_dict, default=repr))
+ def dl(self, name, info, subtitle=False, test=False):
+
+ if test:
+ verbose = self.params.get('verbose')
+ params = {
+ 'test': True,
+ 'quiet': not verbose,
+ 'verbose': verbose,
+ 'noprogress': not verbose,
+ 'nopart': True,
+ 'skip_unavailable_fragments': False,
+ 'keep_fragments': False,
+ 'overwrites': True,
+ '_no_ytdl_file': True,
+ }
+ else:
+ params = self.params
+ fd = get_suitable_downloader(info, params)(self, params)
+ if not test:
+ for ph in self._progress_hooks:
+ fd.add_progress_hook(ph)
+ self.write_debug('Invoking downloader on %r' % info.get('url'))
+ new_info = dict(info)
+ if new_info.get('http_headers') is None:
+ new_info['http_headers'] = self._calc_headers(new_info)
+ return fd.download(name, new_info, subtitle)
+
def process_info(self, info_dict):
"""Process a single resolved IE result."""
self.post_extract(info_dict)
self._num_downloads += 1
- info_dict = self.pre_process(info_dict)
-
# info_dict['_filename'] needs to be set for backward compatibility
info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True)
temp_filename = self.prepare_filename(info_dict, 'temp')
files_to_move = {}
- skip_dl = self.params.get('skip_download', False)
# Forced printings
self.__forced_printings(info_dict, full_filename, incomplete=False)
self.report_error('Cannot write annotations file: ' + annofn)
return
- def dl(name, info, subtitle=False):
- fd = get_suitable_downloader(info, self.params)(self, self.params)
- for ph in self._progress_hooks:
- fd.add_progress_hook(ph)
- if self.params.get('verbose'):
- self.to_screen('[debug] Invoking downloader on %r' % info.get('url'))
- new_info = dict(info)
- if new_info.get('http_headers') is None:
- new_info['http_headers'] = self._calc_headers(new_info)
- return fd.download(name, new_info, subtitle)
-
subtitles_are_requested = any([self.params.get('writesubtitles', False),
self.params.get('writeautomaticsub')])
# ie = self.get_info_extractor(info_dict['extractor_key'])
for sub_lang, sub_info in subtitles.items():
sub_format = sub_info['ext']
- sub_fn = self.prepare_filename(info_dict, 'subtitle')
- sub_filename = subtitles_filename(
- temp_filename if not skip_dl else sub_fn,
- sub_lang, sub_format, info_dict.get('ext'))
- sub_filename_final = subtitles_filename(sub_fn, sub_lang, sub_format, info_dict.get('ext'))
+ sub_filename = subtitles_filename(temp_filename, sub_lang, sub_format, info_dict.get('ext'))
+ sub_filename_final = subtitles_filename(
+ self.prepare_filename(info_dict, 'subtitle'), sub_lang, sub_format, info_dict.get('ext'))
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
sub_info['filepath'] = sub_filename
return
else:
try:
- dl(sub_filename, sub_info.copy(), subtitle=True)
+ self.dl(sub_filename, sub_info.copy(), subtitle=True)
sub_info['filepath'] = sub_filename
files_to_move[sub_filename] = sub_filename_final
- except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ except (ExtractorError, IOError, OSError, ValueError) + network_exceptions as err:
self.report_warning('Unable to download subtitle for "%s": %s' %
(sub_lang, error_to_compat_str(err)))
continue
- if skip_dl:
- if self.params.get('convertsubtitles', False):
- # subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles'))
- filename_real_ext = os.path.splitext(full_filename)[1][1:]
- filename_wo_ext = (
- os.path.splitext(full_filename)[0]
- if filename_real_ext == info_dict['ext']
- else full_filename)
- afilename = '%s.%s' % (filename_wo_ext, self.params.get('convertsubtitles'))
- # if subconv.available:
- # info_dict['__postprocessors'].append(subconv)
- if os.path.exists(encodeFilename(afilename)):
- self.to_screen(
- '[download] %s has already been downloaded and '
- 'converted' % afilename)
- else:
- try:
- self.post_process(full_filename, info_dict, files_to_move)
- except PostProcessingError as err:
- self.report_error('Postprocessing: %s' % str(err))
- return
-
if self.params.get('writeinfojson', False):
infofn = self.prepare_filename(info_dict, 'infojson')
if not self._ensure_dir_exists(encodeFilename(infofn)):
return
info_dict['__infojson_filename'] = infofn
- thumbfn = self.prepare_filename(info_dict, 'thumbnail')
- thumb_fn_temp = temp_filename if not skip_dl else thumbfn
- for thumb_ext in self._write_thumbnails(info_dict, thumb_fn_temp):
- thumb_filename_temp = replace_extension(thumb_fn_temp, thumb_ext, info_dict.get('ext'))
- thumb_filename = replace_extension(thumbfn, thumb_ext, info_dict.get('ext'))
+ for thumb_ext in self._write_thumbnails(info_dict, temp_filename):
+ thumb_filename_temp = replace_extension(temp_filename, thumb_ext, info_dict.get('ext'))
+ thumb_filename = replace_extension(
+ self.prepare_filename(info_dict, 'thumbnail'), thumb_ext, info_dict.get('ext'))
files_to_move[thumb_filename_temp] = thumb_filename
# Write internet shortcut files
if not _write_link_file('desktop', DOT_DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True):
return
- # Download
+ try:
+ info_dict, files_to_move = self.pre_process(info_dict, 'before_dl', files_to_move)
+ except PostProcessingError as err:
+ self.report_error('Preprocessing: %s' % str(err))
+ return
+
must_record_download_archive = False
- if not skip_dl:
+ if self.params.get('skip_download', False):
+ info_dict['filepath'] = temp_filename
+ info_dict['__finaldir'] = os.path.dirname(os.path.abspath(encodeFilename(full_filename)))
+ info_dict['__files_to_move'] = files_to_move
+ info_dict = self.run_pp(MoveFilesAfterDownloadPP(self, False), info_dict)
+ else:
+ # Download
try:
def existing_file(*filepaths):
if not self._ensure_dir_exists(fname):
return
downloaded.append(fname)
- partial_success, real_download = dl(fname, new_info)
+ partial_success, real_download = self.dl(fname, new_info)
info_dict['__real_download'] = info_dict['__real_download'] or real_download
success = success and partial_success
if merger.available and not self.params.get('allow_unplayable_formats'):
# Just a single file
dl_filename = existing_file(full_filename, temp_filename)
if dl_filename is None:
- success, real_download = dl(temp_filename, info_dict)
+ success, real_download = self.dl(temp_filename, info_dict)
info_dict['__real_download'] = real_download
dl_filename = dl_filename or temp_filename
info_dict['__finaldir'] = os.path.dirname(os.path.abspath(encodeFilename(full_filename)))
- except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ except network_exceptions as err:
self.report_error('unable to download video data: %s' % error_to_compat_str(err))
return
except (OSError, IOError) as err:
@staticmethod
def filter_requested_info(info_dict, actually_filter=True):
+ info_dict.pop('__original_infodict', None) # Always remove this
if not actually_filter:
info_dict['epoch'] = int(time.time())
return info_dict
actual_post_extract(video_dict or {})
return
- if '__post_extractor' not in info_dict:
- return
- post_extractor = info_dict['__post_extractor']
- if post_extractor:
- info_dict.update(post_extractor().items())
- del info_dict['__post_extractor']
- return
+ post_extractor = info_dict.get('__post_extractor') or (lambda: {})
+ extra = post_extractor().items()
+ info_dict.update(extra)
+ info_dict.pop('__post_extractor', None)
+
+ original_infodict = info_dict.get('__original_infodict') or {}
+ original_infodict.update(extra)
+ original_infodict.pop('__post_extractor', None)
actual_post_extract(info_dict or {})
- def pre_process(self, ie_info):
+ def pre_process(self, ie_info, key='pre_process', files_to_move=None):
info = dict(ie_info)
- for pp in self._pps['beforedl']:
+ info['__files_to_move'] = files_to_move or {}
+ for pp in self._pps[key]:
info = self.run_pp(pp, info)
- return info
+ return info, info.pop('__files_to_move', None)
def post_process(self, filename, ie_info, files_to_move=None):
"""Run all the postprocessors on the given file."""
info['filepath'] = filename
info['__files_to_move'] = files_to_move or {}
- for pp in ie_info.get('__postprocessors', []) + self._pps['normal']:
+ for pp in ie_info.get('__postprocessors', []) + self._pps['post_process']:
info = self.run_pp(pp, info)
info = self.run_pp(MoveFilesAfterDownloadPP(self), info)
del info['__files_to_move']
- for pp in self._pps['aftermove']:
+ for pp in self._pps['after_move']:
info = self.run_pp(pp, info)
return info
def list_formats(self, info_dict):
formats = info_dict.get('formats', [info_dict])
- new_format = self.params.get('listformats_table', False)
+ new_format = (
+ 'list-formats' not in self.params.get('compat_opts', [])
+ and self.params.get('list_formats_as_table', True) is not False)
if new_format:
table = [
[
'|',
format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes),
format_field(f, 'tbr', '%4dk'),
- f.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n").replace('niconico_', ''),
+ shorten_protocol_name(f.get('protocol', '').replace("native", "n")),
'|',
format_field(f, 'vcodec', default='unknown').replace('none', ''),
format_field(f, 'vbr', '%4dk'),
return
self.to_screen(
'Available %s for %s:' % (name, video_id))
+
+ def _row(lang, formats):
+ exts, names = zip(*((f['ext'], f.get('name', 'unknown')) for f in reversed(formats)))
+ if len(set(names)) == 1:
+ names = [] if names[0] == 'unknown' else names[:1]
+ return [lang, ', '.join(names), ', '.join(exts)]
+
self.to_screen(render_table(
- ['Language', 'formats'],
- [[lang, ', '.join(f['ext'] for f in reversed(formats))]
- for lang, formats in subtitles.items()]))
+ ['Language', 'Name', 'Formats'],
+ [_row(lang, formats) for lang, formats in subtitles.items()],
+ hideEmpty=True))
def urlopen(self, req):
""" Start an HTTP download """
if _PLUGIN_CLASSES:
self._write_string(
'[debug] Plugin Extractors: %s\n' % [ie.ie_key() for ie in _PLUGIN_CLASSES])
+ if self.params.get('compat_opts'):
+ self._write_string(
+ '[debug] Compatibility options: %s\n' % ', '.join(self.params.get('compat_opts')))
try:
sp = subprocess.Popen(
['git', 'rev-parse', '--short', 'HEAD'],
ret.append(suffix + thumb_ext)
self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
(info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
- except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ except network_exceptions as err:
self.report_warning('Unable to download thumbnail "%s": %s' %
(t['url'], error_to_compat_str(err)))
if ret and not write_all:
projects / yt-dlp.git / blobdiff