compat_kwargs,
compat_numeric_types,
compat_os_name,
+ compat_shlex_quote,
compat_str,
compat_tokenize_tokenize,
compat_urllib_error,
float_or_none,
format_bytes,
format_field,
- STR_FORMAT_RE,
+ STR_FORMAT_RE_TMPL,
+ STR_FORMAT_TYPES,
formatSeconds,
GeoRestrictedError,
HEADRequest,
try_get,
UnavailableVideoError,
url_basename,
+ variadic,
version_tuple,
write_json_file,
write_string,
)
from .extractor.openload import PhantomJSwrapper
from .downloader import (
+ FFmpegFD,
get_suitable_downloader,
shorten_protocol_name
)
(or video) as a single JSON line.
force_write_download_archive: Force writing download archive regardless
of 'skip_download' or 'simulate'.
- simulate: Do not download the video files.
+ simulate: Do not download the video files. If unset (or None),
+ simulate only if listsubtitles, listformats or list_thumbnails is used
format: Video format code. see "FORMAT SELECTION" for more details.
allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
ignore_no_formats_error: Ignore "No video formats" error. Usefull for
writedesktoplink: Write a Linux internet shortcut file (.desktop)
writesubtitles: Write the video subtitles to a file
writeautomaticsub: Write the automatically generated subtitles to a file
- allsubtitles: Deprecated - Use subtitlelangs = ['all']
+ allsubtitles: Deprecated - Use subtitleslangs = ['all']
Downloads all the subtitles of the video
(requires writesubtitles or writeautomaticsub)
listsubtitles: Lists all available subtitles for the video
progress, with a dictionary with the entries
* status: One of "downloading", "error", or "finished".
Check this first and ignore unknown values.
+ * info_dict: The extracted info_dict
If status is one of "downloading", or "finished", the
following properties may also be present:
compat_opts: Compatibility options. See "Differences in default behavior".
The following options do not work when used through the API:
filename, abort-on-error, multistreams, no-live-chat,
- no-playlist-metafiles. Refer __init__.py for their implementation
+ no-clean-infojson, no-playlist-metafiles, no-keep-subs.
+ Refer __init__.py for their implementation
The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see yt_dlp/downloader/common.py):
def save_console_title(self):
if not self.params.get('consoletitle', False):
return
- if self.params.get('simulate', False):
+ if self.params.get('simulate'):
return
if compat_os_name != 'nt' and 'TERM' in os.environ:
# Save the title on stack
def restore_console_title(self):
if not self.params.get('consoletitle', False):
return
- if self.params.get('simulate', False):
+ if self.params.get('simulate'):
return
if compat_os_name != 'nt' and 'TERM' in os.environ:
# Restore the title from stack
return sanitize_path(path, force=self.params.get('windowsfilenames'))
@staticmethod
- def validate_outtmpl(tmpl):
+ def _outtmpl_expandpath(outtmpl):
+ # expand_path translates '%%' into '%' and '$$' into '$'
+ # correspondingly that is not what we want since we need to keep
+ # '%%' intact for template dict substitution step. Working around
+ # with boundary-alike separator hack.
+ sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
+ outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
+
+ # outtmpl should be expand_path'ed before template dict substitution
+ # because meta fields may contain env variables we don't want to
+ # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
+ # title "Hello $PATH", we don't want `$PATH` to be expanded.
+ return expand_path(outtmpl).replace(sep, '')
+
+ @staticmethod
+ def escape_outtmpl(outtmpl):
+ ''' Escape any remaining strings like %s, %abc% etc. '''
+ return re.sub(
+ STR_FORMAT_RE_TMPL.format('', '(?![%(\0])'),
+ lambda mobj: ('' if mobj.group('has_key') else '%') + mobj.group(0),
+ outtmpl)
+
+ @classmethod
+ def validate_outtmpl(cls, outtmpl):
''' @return None or Exception object '''
+ outtmpl = re.sub(
+ STR_FORMAT_RE_TMPL.format('[^)]*', '[ljq]'),
+ lambda mobj: f'{mobj.group(0)[:-1]}s',
+ cls._outtmpl_expandpath(outtmpl))
try:
- re.sub(
- STR_FORMAT_RE.format(''),
- lambda mobj: ('%' if not mobj.group('has_key') else '') + mobj.group(0),
- tmpl
- ) % collections.defaultdict(int)
+ cls.escape_outtmpl(outtmpl) % collections.defaultdict(int)
return None
except ValueError as err:
return err
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
- """ Make the template and info_dict suitable for substitution (outtmpl % info_dict)"""
- info_dict = dict(info_dict)
+ """ Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """
+ info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
na = self.params.get('outtmpl_na_placeholder', 'NA')
+ info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList
+ for key in ('__original_infodict', '__postprocessors'):
+ info_dict.pop(key, None)
info_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
formatSeconds(info_dict['duration'], '-' if sanitize else ':')
if info_dict.get('duration', None) is not None
else None)
- info_dict['epoch'] = int(time.time())
info_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
if info_dict.get('resolution') is None:
info_dict['resolution'] = self.format_resolution(info_dict, default=None)
}
TMPL_DICT = {}
- EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE.format('[^)]*'))
+ EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljq]'))
MATH_FUNCTIONS = {
'+': float.__add__,
'-': float.__sub__,
}
# Field is of the form key1.key2...
# where keys (except first) can be string, int or slice
- FIELD_RE = r'\w+(?:\.(?:\w+|{num}|{num}?(?::{num}?){{1,2}}))*'.format(num=r'(?:-?\d+)')
+ FIELD_RE = r'\w*(?:\.(?:\w+|{num}|{num}?(?::{num}?){{1,2}}))*'.format(num=r'(?:-?\d+)')
MATH_FIELD_RE = r'''{field}|{num}'''.format(field=FIELD_RE, num=r'-?\d+(?:.\d+)?')
MATH_OPERATORS_RE = r'(?:%s)' % '|'.join(map(re.escape, MATH_FUNCTIONS.keys()))
INTERNAL_FORMAT_RE = re.compile(r'''(?x)
(?:\|(?P<default>.*?))?
$'''.format(field=FIELD_RE, math_op=MATH_OPERATORS_RE, math_field=MATH_FIELD_RE))
- get_key = lambda k: traverse_obj(
- info_dict, k.split('.'), is_user_input=True, traverse_string=True)
+ def _traverse_infodict(k):
+ k = k.split('.')
+ if k[0] == '':
+ k.pop(0)
+ return traverse_obj(info_dict, k, is_user_input=True, traverse_string=True)
def get_value(mdict):
# Object traversal
- value = get_key(mdict['fields'])
+ value = _traverse_infodict(mdict['fields'])
# Negative
if mdict['negate']:
value = float_or_none(value)
item, multiplier = (item[1:], -1) if item[0] == '-' else (item, 1)
offset = float_or_none(item)
if offset is None:
- offset = float_or_none(get_key(item))
+ offset = float_or_none(_traverse_infodict(item))
try:
value = operator(value, multiplier * offset)
except (TypeError, ZeroDivisionError):
return value
+ def _dumpjson_default(obj):
+ if isinstance(obj, (set, LazyList)):
+ return list(obj)
+ raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
+
def create_key(outer_mobj):
if not outer_mobj.group('has_key'):
- return '%{}'.format(outer_mobj.group(0))
+ return f'%{outer_mobj.group(0)}'
+ prefix = outer_mobj.group('prefix')
key = outer_mobj.group('key')
- fmt = outer_mobj.group('format')
+ original_fmt = fmt = outer_mobj.group('format')
mobj = re.match(INTERNAL_FORMAT_RE, key)
if mobj is None:
value, default, mobj = None, na, {'fields': ''}
value = default if value is None else value
- if fmt == 'c':
- value = compat_str(value)
+ str_fmt = f'{fmt[:-1]}s'
+ if fmt[-1] == 'l':
+ value, fmt = ', '.join(variadic(value)), str_fmt
+ elif fmt[-1] == 'j':
+ value, fmt = json.dumps(value, default=_dumpjson_default), str_fmt
+ elif fmt[-1] == 'q':
+ value, fmt = compat_shlex_quote(str(value)), str_fmt
+ elif fmt[-1] == 'c':
+ value = str(value)
if value is None:
value, fmt = default, 's'
else:
value = float_or_none(value)
if value is None:
value, fmt = default, 's'
+
if sanitize:
if fmt[-1] == 'r':
# If value is an object, sanitize might convert it to a string
# So we convert it to repr first
- value, fmt = repr(value), '%ss' % fmt[:-1]
+ value, fmt = repr(value), str_fmt
if fmt[-1] in 'csr':
value = sanitize(mobj['fields'].split('.')[-1], value)
- key += '\0%s' % fmt
+
+ key = '%s\0%s' % (key.replace('%', '%\0'), original_fmt)
TMPL_DICT[key] = value
- return '%({key}){fmt}'.format(key=key, fmt=fmt)
+ return f'{prefix}%({key}){fmt}'
return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT
is_id=(k == 'id' or k.endswith('_id')))
outtmpl = self.outtmpl_dict.get(tmpl_type, self.outtmpl_dict['default'])
outtmpl, template_dict = self.prepare_outtmpl(outtmpl, info_dict, sanitize)
-
- # expand_path translates '%%' into '%' and '$$' into '$'
- # correspondingly that is not what we want since we need to keep
- # '%%' intact for template dict substitution step. Working around
- # with boundary-alike separator hack.
- sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
- outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
-
- # outtmpl should be expand_path'ed before template dict substitution
- # because meta fields may contain env variables we don't want to
- # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
- # title "Hello $PATH", we don't want `$PATH` to be expanded.
- filename = expand_path(outtmpl).replace(sep, '') % template_dict
+ outtmpl = self.escape_outtmpl(self._outtmpl_expandpath(outtmpl))
+ filename = outtmpl % template_dict
force_ext = OUTTMPL_TYPES.get(tmpl_type)
if force_ext is not None:
else:
self.to_screen('[info] Writing playlist metadata as JSON to: ' + infofn)
try:
- write_json_file(self.filter_requested_info(ie_result, self.params.get('clean_infojson', True)), infofn)
+ write_json_file(self.sanitize_info(ie_result, self.params.get('clean_infojson', True)), infofn)
except (OSError, IOError):
self.report_error('Cannot write playlist metadata to JSON file ' + infofn)
return merger.available and merger.can_merge()
prefer_best = (
- not self.params.get('simulate', False)
+ not self.params.get('simulate')
and download
and (
not can_merge()
if not allow_multiple_streams[aud_vid] and fmt_info.get(aud_vid[0] + 'codec') != 'none':
if get_no_more[aud_vid]:
formats_info.pop(i)
+ break
get_no_more[aud_vid] = True
if len(formats_info) == 1:
info_dict, _ = self.pre_process(info_dict)
- list_only = self.params.get('list_thumbnails') or self.params.get('listformats') or self.params.get('listsubtitles')
+ if self.params.get('list_thumbnails'):
+ self.list_thumbnails(info_dict)
+ if self.params.get('listformats'):
+ if not info_dict.get('formats'):
+ raise ExtractorError('No video formats found', expected=True)
+ self.list_formats(info_dict)
+ if self.params.get('listsubtitles'):
+ if 'automatic_captions' in info_dict:
+ self.list_subtitles(
+ info_dict['id'], automatic_captions, 'automatic captions')
+ self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
+ list_only = self.params.get('simulate') is None and (
+ self.params.get('list_thumbnails') or self.params.get('listformats') or self.params.get('listsubtitles'))
if list_only:
+ # Without this printing, -F --print-json will not work
self.__forced_printings(info_dict, self.prepare_filename(info_dict), incomplete=True)
- if self.params.get('list_thumbnails'):
- self.list_thumbnails(info_dict)
- if self.params.get('listformats'):
- if not info_dict.get('formats'):
- raise ExtractorError('No video formats found', expected=True)
- self.list_formats(info_dict)
- if self.params.get('listsubtitles'):
- if 'automatic_captions' in info_dict:
- self.list_subtitles(
- info_dict['id'], automatic_captions, 'automatic captions')
- self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
return
format_selector = self.format_selector
elif 'url' in info_dict:
info_dict['urls'] = info_dict['url'] + info_dict.get('play_path', '')
+ if self.params.get('forceprint') or self.params.get('forcejson'):
+ self.post_extract(info_dict)
for tmpl in self.params.get('forceprint', []):
if re.match(r'\w+$', tmpl):
tmpl = '%({})s'.format(tmpl)
tmpl, info_copy = self.prepare_outtmpl(tmpl, info_dict)
- self.to_stdout(tmpl % info_copy)
+ self.to_stdout(self.escape_outtmpl(tmpl) % info_copy)
print_mandatory('title')
print_mandatory('id')
self.to_stdout(formatSeconds(info_dict['duration']))
print_mandatory('format')
- if self.params.get('forcejson', False):
- self.post_extract(info_dict)
- self.to_stdout(json.dumps(info_dict, default=repr))
+ if self.params.get('forcejson'):
+ self.to_stdout(json.dumps(self.sanitize_info(info_dict)))
def dl(self, name, info, subtitle=False, test=False):
}
else:
params = self.params
- fd = get_suitable_downloader(info, params)(self, params)
+ fd = get_suitable_downloader(info, params, to_stdout=(name == '-'))(self, params)
if not test:
for ph in self._progress_hooks:
fd.add_progress_hook(ph)
# Forced printings
self.__forced_printings(info_dict, full_filename, incomplete=('format' not in info_dict))
- if self.params.get('simulate', False):
+ if self.params.get('simulate'):
if self.params.get('force_write_download_archive', False):
self.record_download_archive(info_dict)
else:
self.to_screen('[info] Writing video metadata as JSON to: ' + infofn)
try:
- write_json_file(self.filter_requested_info(info_dict, self.params.get('clean_infojson', True)), infofn)
+ write_json_file(self.sanitize_info(info_dict, self.params.get('clean_infojson', True)), infofn)
except (OSError, IOError):
self.report_error('Cannot write video metadata to JSON file ' + infofn)
return
info_dict['ext'] = 'mkv'
self.report_warning(
'Requested formats are incompatible for merge and will be merged into mkv.')
+ new_ext = info_dict['ext']
- def correct_ext(filename):
+ def correct_ext(filename, ext=new_ext):
+ if filename == '-':
+ return filename
filename_real_ext = os.path.splitext(filename)[1][1:]
filename_wo_ext = (
os.path.splitext(filename)[0]
- if filename_real_ext == old_ext
+ if filename_real_ext in (old_ext, new_ext)
else filename)
- return '%s.%s' % (filename_wo_ext, info_dict['ext'])
+ return '%s.%s' % (filename_wo_ext, ext)
# Ensure filename always has a correct extension for successful merge
full_filename = correct_ext(full_filename)
info_dict['__real_download'] = False
_protocols = set(determine_protocol(f) for f in requested_formats)
- if len(_protocols) == 1:
+ if len(_protocols) == 1: # All requested formats have same protocol
info_dict['protocol'] = _protocols.pop()
- directly_mergable = (
- 'no-direct-merge' not in self.params.get('compat_opts', [])
- and info_dict.get('protocol') is not None # All requested formats have same protocol
- and not self.params.get('allow_unplayable_formats')
- and get_suitable_downloader(info_dict, self.params).__name__ == 'FFmpegFD')
- if directly_mergable:
- info_dict['url'] = requested_formats[0]['url']
- # Treat it as a single download
- dl_filename = existing_file(full_filename, temp_filename)
- if dl_filename is None:
- success, real_download = self.dl(temp_filename, info_dict)
- info_dict['__real_download'] = real_download
+ directly_mergable = FFmpegFD.can_merge_formats(info_dict)
+ if dl_filename is not None:
+ pass
+ elif (directly_mergable and get_suitable_downloader(
+ info_dict, self.params, to_stdout=(temp_filename == '-')) == FFmpegFD):
+ info_dict['url'] = '\n'.join(f['url'] for f in requested_formats)
+ success, real_download = self.dl(temp_filename, info_dict)
+ info_dict['__real_download'] = real_download
else:
downloaded = []
merger = FFmpegMergerPP(self)
'You have requested merging of multiple formats but ffmpeg is not installed. '
'The formats won\'t be merged.')
- if dl_filename is None:
- for f in requested_formats:
- new_info = dict(info_dict)
- del new_info['requested_formats']
- new_info.update(f)
+ if temp_filename == '-':
+ reason = ('using a downloader other than ffmpeg' if directly_mergable
+ else 'but the formats are incompatible for simultaneous download' if merger.available
+ else 'but ffmpeg is not installed')
+ self.report_warning(
+ f'You have requested downloading multiple formats to stdout {reason}. '
+ 'The formats will be streamed one after the other')
+ fname = temp_filename
+ for f in requested_formats:
+ new_info = dict(info_dict)
+ del new_info['requested_formats']
+ new_info.update(f)
+ if temp_filename != '-':
fname = prepend_extension(
- self.prepare_filename(new_info, 'temp'),
+ correct_ext(temp_filename, new_info['ext']),
'f%s' % f['format_id'], new_info['ext'])
if not self._ensure_dir_exists(fname):
return
downloaded.append(fname)
- partial_success, real_download = self.dl(fname, new_info)
- info_dict['__real_download'] = info_dict['__real_download'] or real_download
- success = success and partial_success
- if merger.available and not self.params.get('allow_unplayable_formats'):
- info_dict['__postprocessors'].append(merger)
- info_dict['__files_to_merge'] = downloaded
- # Even if there were no downloads, it is being merged only now
- info_dict['__real_download'] = True
- else:
- for file in downloaded:
- files_to_move[file] = None
+ partial_success, real_download = self.dl(fname, new_info)
+ info_dict['__real_download'] = info_dict['__real_download'] or real_download
+ success = success and partial_success
+ if merger.available and not self.params.get('allow_unplayable_formats'):
+ info_dict['__postprocessors'].append(merger)
+ info_dict['__files_to_merge'] = downloaded
+ # Even if there were no downloads, it is being merged only now
+ info_dict['__real_download'] = True
+ else:
+ for file in downloaded:
+ files_to_move[file] = None
else:
# Just a single file
dl_filename = existing_file(full_filename, temp_filename)
else:
if self.params.get('dump_single_json', False):
self.post_extract(res)
- self.to_stdout(json.dumps(res, default=repr))
+ self.to_stdout(json.dumps(self.sanitize_info(res)))
return self._download_retcode
[info_filename], mode='r',
openhook=fileinput.hook_encoded('utf-8'))) as f:
# FileInput doesn't have a read method, we can't call json.load
- info = self.filter_requested_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True))
+ info = self.sanitize_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True))
try:
self.process_ie_result(info, download=True)
except (DownloadError, EntryNotInPlaylist, ThrottledDownload):
return self._download_retcode
@staticmethod
- def filter_requested_info(info_dict, actually_filter=True):
- remove_keys = ['__original_infodict'] # Always remove this since this may contain a copy of the entire dict
+ def sanitize_info(info_dict, remove_private_keys=False):
+ ''' Sanitize the infodict for converting to json '''
+ info_dict.setdefault('epoch', int(time.time()))
+ remove_keys = {'__original_infodict'} # Always remove this since this may contain a copy of the entire dict
keep_keys = ['_type'], # Always keep this to facilitate load-info-json
- if actually_filter:
- remove_keys += ('requested_formats', 'requested_subtitles', 'requested_entries', 'filepath', 'entries', 'original_url')
+ if remove_private_keys:
+ remove_keys |= {
+ 'requested_formats', 'requested_subtitles', 'requested_entries',
+ 'filepath', 'entries', 'original_url', 'playlist_autonumber',
+ }
empty_values = (None, {}, [], set(), tuple())
reject = lambda k, v: k not in keep_keys and (
k.startswith('_') or k in remove_keys or v in empty_values)
else:
- info_dict['epoch'] = int(time.time())
reject = lambda k, v: k in remove_keys
filter_fn = lambda obj: (
list(map(filter_fn, obj)) if isinstance(obj, (LazyList, list, tuple, set))
else dict((k, filter_fn(v)) for k, v in obj.items() if not reject(k, v)))
return filter_fn(info_dict)
+ @staticmethod
+ def filter_requested_info(info_dict, actually_filter=True):
+ ''' Alias of sanitize_info for backward compatibility '''
+ return YoutubeDL.sanitize_info(info_dict, actually_filter)
+
def run_pp(self, pp, infodict):
files_to_delete = []
if '__files_to_move' not in infodict:
format_field(f, 'language', '[%s]'),
format_field(f, 'format_note'),
format_field(f, 'container', ignore=(None, f.get('ext'))),
- format_field(f, 'asr', '%5dHz')))),
+ ))),
] for f in formats if f.get('preference') is None or f['preference'] >= -1000]
header_line = ['ID', 'EXT', 'RESOLUTION', 'FPS', '|', ' FILESIZE', ' TBR', 'PROTO',
'|', 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'MORE INFO']
projects / yt-dlp.git / blobdiff