import random
from string import ascii_letters
+from zipimport import zipimporter
from .compat import (
compat_basestring,
date_from_str,
DateRange,
DEFAULT_OUTTMPL,
+ OUTTMPL_TYPES,
determine_ext,
determine_protocol,
DOT_DESKTOP_LINK_TEMPLATE,
ExistingVideoReached,
expand_path,
ExtractorError,
+ float_or_none,
format_bytes,
format_field,
formatSeconds,
sanitized_Request,
std_headers,
str_or_none,
+ strftime_or_none,
subtitles_filename,
to_high_limit_path,
UnavailableVideoError,
forcejson: Force printing info_dict as JSON.
dump_single_json: Force printing the info_dict of the whole playlist
(or video) as a single JSON line.
- force_write_download_archive: Force writing download archive regardless of
- 'skip_download' or 'simulate'.
+ force_write_download_archive: Force writing download archive regardless
+ of 'skip_download' or 'simulate'.
simulate: Do not download the video files.
format: Video format code. see "FORMAT SELECTION" for more details.
- format_sort: How to sort the video formats. see "Sorting Formats" for more details.
- format_sort_force: Force the given format_sort. see "Sorting Formats" for more details.
- allow_multiple_video_streams: Allow multiple video streams to be merged into a single file
- allow_multiple_audio_streams: Allow multiple audio streams to be merged into a single file
- outtmpl: Template for output names.
+ allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
+ format_sort: How to sort the video formats. see "Sorting Formats"
+ for more details.
+ format_sort_force: Force the given format_sort. see "Sorting Formats"
+ for more details.
+ allow_multiple_video_streams: Allow multiple video streams to be merged
+ into a single file
+ allow_multiple_audio_streams: Allow multiple audio streams to be merged
+ into a single file
+ paths: Dictionary of output paths. The allowed keys are 'home'
+ 'temp' and the keys of OUTTMPL_TYPES (in utils.py)
+ outtmpl: Dictionary of templates for output names. Allowed keys
+ are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
+ A string a also accepted for backward compatibility
outtmpl_na_placeholder: Placeholder for unavailable meta fields.
restrictfilenames: Do not allow "&" and spaces in file names
trim_file_name: Limit length of filename (extension excluded)
+ windowsfilenames: Force the filenames to be windows compatible
ignoreerrors: Do not stop on download errors
(Default True when running youtube-dlc,
but False when directly accessing YoutubeDL class)
unless writeinfojson is also given
writeannotations: Write the video annotations to a .annotations.xml file
writethumbnail: Write the thumbnail image to a file
+ allow_playlist_files: Whether to write playlists' description, infojson etc
+ also to disk when using the 'write*' options
write_all_thumbnails: Write all thumbnail formats to files
writelink: Write an internet shortcut file, depending on the
current platform (.url/.webloc/.desktop)
Progress hooks are guaranteed to be called at least once
(with status "finished") if the download is successful.
merge_output_format: Extension to use when merging formats.
+ final_ext: Expected final extension; used to detect when the file was
+ already downloaded and converted. "merge_output_format" is
+ replaced by this extension when given
fixup: Automatically correct known faults of the file.
One of:
- "never": do nothing
postprocessor/executable. The dict can also have "PP+EXE" keys
which are used when the given exe is used by the given PP.
Use 'default' as the name for arguments to passed to all PP
- The following options are used by the Youtube extractor:
+
+ The following options are used by the extractors:
+ dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
+ hls_split_discontinuity: Split HLS playlists to different formats at
+ discontinuities such as ad breaks (default: False)
youtube_include_dash_manifest: If True (default), DASH manifests and related
data will be downloaded and processed by extractor.
You can reduce network I/O by disabling it if you don't
- care about DASH.
+ care about DASH. (only for youtube)
+ youtube_include_hls_manifest: If True (default), HLS manifests and related
+ data will be downloaded and processed by extractor.
+ You can reduce network I/O by disabling it if you don't
+ care about HLS. (only for youtube)
"""
_NUMERIC_FIELDS = set((
if self.params.get('geo_verification_proxy') is None:
self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
+ if self.params.get('final_ext'):
+ if self.params.get('merge_output_format'):
+ self.report_warning('--merge-output-format will be ignored since --remux-video or --recode-video is given')
+ self.params['merge_output_format'] = self.params['final_ext']
+
+ if 'overwrites' in self.params and self.params['overwrites'] is None:
+ del self.params['overwrites']
+
check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
'Set the LC_ALL environment variable to fix this.')
self.params['restrictfilenames'] = True
- if isinstance(params.get('outtmpl'), bytes):
- self.report_warning(
- 'Parameter outtmpl is bytes, but should be a unicode string. '
- 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
+ self.outtmpl_dict = self.parse_outtmpl()
self._setup_opener()
def report_file_delete(self, file_name):
"""Report that existing file will be deleted."""
try:
- self.to_screen('Deleting already existent file %s' % file_name)
+ self.to_screen('Deleting existing file %s' % file_name)
except UnicodeEncodeError:
- self.to_screen('Deleting already existent file')
+ self.to_screen('Deleting existing file')
+
+ def parse_outtmpl(self):
+ outtmpl_dict = self.params.get('outtmpl', {})
+ if not isinstance(outtmpl_dict, dict):
+ outtmpl_dict = {'default': outtmpl_dict}
+ outtmpl_dict.update({
+ k: v for k, v in DEFAULT_OUTTMPL.items()
+ if not outtmpl_dict.get(k)})
+ for key, val in outtmpl_dict.items():
+ if isinstance(val, bytes):
+ self.report_warning(
+ 'Parameter outtmpl is bytes, but should be a unicode string. '
+ 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
+ return outtmpl_dict
- def prepare_filename(self, info_dict, warn=False):
- """Generate the output filename."""
+ def _prepare_filename(self, info_dict, tmpl_type='default'):
try:
template_dict = dict(info_dict)
+ template_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
+ formatSeconds(info_dict['duration'], '-')
+ if info_dict.get('duration', None) is not None
+ else None)
+
template_dict['epoch'] = int(time.time())
autonumber_size = self.params.get('autonumber_size')
if autonumber_size is None:
template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
for k, v in template_dict.items()
if v is not None and not isinstance(v, (list, tuple, dict)))
- template_dict = collections.defaultdict(lambda: self.params.get('outtmpl_na_placeholder', 'NA'), template_dict)
+ na = self.params.get('outtmpl_na_placeholder', 'NA')
+ template_dict = collections.defaultdict(lambda: na, template_dict)
- outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
+ outtmpl = self.outtmpl_dict.get(tmpl_type, self.outtmpl_dict['default'])
+ force_ext = OUTTMPL_TYPES.get(tmpl_type)
# For fields playlist_index and autonumber convert all occurrences
# of %(field)s to %(field)0Nd for backward compatibility
r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
outtmpl)
+ # As of [1] format syntax is:
+ # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
+ # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
+ FORMAT_RE = r'''(?x)
+ (?<!%)
+ %
+ \({0}\) # mapping key
+ (?:[#0\-+ ]+)? # conversion flags (optional)
+ (?:\d+)? # minimum field width (optional)
+ (?:\.\d+)? # precision (optional)
+ [hlL]? # length modifier (optional)
+ (?P<type>[diouxXeEfFgGcrs%]) # conversion type
+ '''
+
+ numeric_fields = list(self._NUMERIC_FIELDS)
+
+ # Format date
+ FORMAT_DATE_RE = FORMAT_RE.format(r'(?P<key>(?P<field>\w+)>(?P<format>.+?))')
+ for mobj in re.finditer(FORMAT_DATE_RE, outtmpl):
+ conv_type, field, frmt, key = mobj.group('type', 'field', 'format', 'key')
+ if key in template_dict:
+ continue
+ value = strftime_or_none(template_dict.get(field), frmt, na)
+ if conv_type in 'crs': # string
+ value = sanitize(field, value)
+ else: # number
+ numeric_fields.append(key)
+ value = float_or_none(value, default=None)
+ if value is not None:
+ template_dict[key] = value
+
# Missing numeric fields used together with integer presentation types
# in format specification will break the argument substitution since
# string NA placeholder is returned for missing fields. We will patch
# output template for missing fields to meet string presentation type.
- for numeric_field in self._NUMERIC_FIELDS:
+ for numeric_field in numeric_fields:
if numeric_field not in template_dict:
- # As of [1] format syntax is:
- # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
- # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
- FORMAT_RE = r'''(?x)
- (?<!%)
- %
- \({0}\) # mapping key
- (?:[#0\-+ ]+)? # conversion flags (optional)
- (?:\d+)? # minimum field width (optional)
- (?:\.\d+)? # precision (optional)
- [hlL]? # length modifier (optional)
- [diouxXeEfFgGcrs%] # conversion type
- '''
outtmpl = re.sub(
- FORMAT_RE.format(numeric_field),
+ FORMAT_RE.format(re.escape(numeric_field)),
r'%({0})s'.format(numeric_field), outtmpl)
# expand_path translates '%%' into '%' and '$$' into '$'
# title "Hello $PATH", we don't want `$PATH` to be expanded.
filename = expand_path(outtmpl).replace(sep, '') % template_dict
+ if force_ext is not None:
+ filename = replace_extension(filename, force_ext, template_dict.get('ext'))
+
# https://github.com/blackjack4494/youtube-dlc/issues/85
trim_file_name = self.params.get('trim_file_name', False)
if trim_file_name:
sub_ext = fn_groups[-2]
filename = '.'.join(filter(None, [fn_groups[0][:trim_file_name], sub_ext, ext]))
- # Temporary fix for #4787
- # 'Treat' all problem characters by passing filename through preferredencoding
- # to workaround encoding issues with subprocess on python2 @ Windows
- if sys.version_info < (3, 0) and sys.platform == 'win32':
- filename = encodeFilename(filename, True).decode(preferredencoding())
- filename = sanitize_path(filename)
-
- if warn and not self.__prepare_filename_warned:
- if not self.params.get('paths'):
- pass
- elif filename == '-':
- self.report_warning('--paths is ignored when an outputting to stdout')
- elif os.path.isabs(filename):
- self.report_warning('--paths is ignored since an absolute path is given in output template')
- self.__prepare_filename_warned = True
-
return filename
except ValueError as err:
self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
return None
- def prepare_filepath(self, filename, dir_type=''):
- if filename == '-':
- return filename
+ def prepare_filename(self, info_dict, dir_type='', warn=False):
+ """Generate the output filename."""
paths = self.params.get('paths', {})
assert isinstance(paths, dict)
+ filename = self._prepare_filename(info_dict, dir_type or 'default')
+
+ if warn and not self.__prepare_filename_warned:
+ if not paths:
+ pass
+ elif filename == '-':
+ self.report_warning('--paths is ignored when an outputting to stdout')
+ elif os.path.isabs(filename):
+ self.report_warning('--paths is ignored since an absolute path is given in output template')
+ self.__prepare_filename_warned = True
+ if filename == '-' or not filename:
+ return filename
+
homepath = expand_path(paths.get('home', '').strip())
assert isinstance(homepath, compat_str)
subdir = expand_path(paths.get(dir_type, '').strip()) if dir_type else ''
assert isinstance(subdir, compat_str)
- return sanitize_path(os.path.join(homepath, subdir, filename))
+ path = os.path.join(homepath, subdir, filename)
+
+ # Temporary fix for #4787
+ # 'Treat' all problem characters by passing filename through preferredencoding
+ # to workaround encoding issues with subprocess on python2 @ Windows
+ if sys.version_info < (3, 0) and sys.platform == 'win32':
+ path = encodeFilename(path, True).decode(preferredencoding())
+ return sanitize_path(path, force=self.params.get('windowsfilenames'))
def _match_entry(self, info_dict, incomplete):
""" Returns None if the file should be downloaded """
self.add_extra_info(ie_result, {
'extractor': ie.IE_NAME,
'webpage_url': url,
- 'duration_string': (
- formatSeconds(ie_result['duration'], '-')
- if ie_result.get('duration', None) is not None
- else None),
'webpage_url_basename': url_basename(url),
'extractor_key': ie.ie_key(),
})
extract_flat = self.params.get('extract_flat', False)
if ((extract_flat == 'in_playlist' and 'playlist' in extra_info)
or extract_flat is True):
- self.__forced_printings(
- ie_result,
- self.prepare_filepath(self.prepare_filename(ie_result)),
- incomplete=True)
+ self.__forced_printings(ie_result, self.prepare_filename(ie_result), incomplete=True)
return ie_result
if result_type == 'video':
playlist = ie_result.get('title') or ie_result.get('id')
self.to_screen('[download] Downloading playlist: %s' % playlist)
- def ensure_dir_exists(path):
- return make_dir(path, self.report_error)
+ if self.params.get('allow_playlist_files', True):
+ ie_copy = {
+ 'playlist': playlist,
+ 'playlist_id': ie_result.get('id'),
+ 'playlist_title': ie_result.get('title'),
+ 'playlist_uploader': ie_result.get('uploader'),
+ 'playlist_uploader_id': ie_result.get('uploader_id'),
+ 'playlist_index': 0
+ }
+ ie_copy.update(dict(ie_result))
- if self.params.get('writeinfojson', False):
- infofn = replace_extension(
- self.prepare_filepath(self.prepare_filename(ie_result), 'infojson'),
- 'info.json', ie_result.get('ext'))
- if not ensure_dir_exists(encodeFilename(infofn)):
- return
- if self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
- self.to_screen('[info] Playlist description metadata is already present')
- else:
- self.to_screen('[info] Writing description playlist metadata as JSON to: ' + infofn)
- playlist_info = dict(ie_result)
- playlist_info.pop('entries')
- try:
- write_json_file(self.filter_requested_info(playlist_info), infofn)
- except (OSError, IOError):
- self.report_error('Cannot write playlist description metadata to JSON file ' + infofn)
+ def ensure_dir_exists(path):
+ return make_dir(path, self.report_error)
- if self.params.get('writedescription', False):
- descfn = replace_extension(
- self.prepare_filepath(self.prepare_filename(ie_result), 'description'),
- 'description', ie_result.get('ext'))
- if not ensure_dir_exists(encodeFilename(descfn)):
- return
- if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
- self.to_screen('[info] Playlist description is already present')
- elif ie_result.get('description') is None:
- self.report_warning('There\'s no playlist description to write.')
- else:
- try:
- self.to_screen('[info] Writing playlist description to: ' + descfn)
- with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
- descfile.write(ie_result['description'])
- except (OSError, IOError):
- self.report_error('Cannot write playlist description file ' + descfn)
+ if self.params.get('writeinfojson', False):
+ infofn = self.prepare_filename(ie_copy, 'pl_infojson')
+ if not ensure_dir_exists(encodeFilename(infofn)):
return
+ if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
+ self.to_screen('[info] Playlist metadata is already present')
+ else:
+ playlist_info = dict(ie_result)
+ # playlist_info['entries'] = list(playlist_info['entries']) # Entries is a generator which shouldnot be resolved here
+ del playlist_info['entries']
+ self.to_screen('[info] Writing playlist metadata as JSON to: ' + infofn)
+ try:
+ write_json_file(self.filter_requested_info(playlist_info), infofn)
+ except (OSError, IOError):
+ self.report_error('Cannot write playlist metadata to JSON file ' + infofn)
+
+ if self.params.get('writedescription', False):
+ descfn = self.prepare_filename(ie_copy, 'pl_description')
+ if not ensure_dir_exists(encodeFilename(descfn)):
+ return
+ if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
+ self.to_screen('[info] Playlist description is already present')
+ elif ie_result.get('description') is None:
+ self.report_warning('There\'s no playlist description to write.')
+ else:
+ try:
+ self.to_screen('[info] Writing playlist description to: ' + descfn)
+ with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
+ descfile.write(ie_result['description'])
+ except (OSError, IOError):
+ self.report_error('Cannot write playlist description file ' + descfn)
+ return
playlist_results = []
and (
not can_merge()
or info_dict.get('is_live', False)
- or self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-'))
+ or self.outtmpl_dict['default'] == '-'))
return (
'best/bestvideo+bestaudio'
if req_format is None:
req_format = self._default_format_spec(info_dict, download=download)
if self.params.get('verbose'):
- self._write_string('[debug] Default format spec: %s\n' % req_format)
+ self.to_screen('[debug] Default format spec: %s' % req_format)
format_selector = self.build_format_selector(req_format)
info_dict = self.pre_process(info_dict)
- filename = self.prepare_filename(info_dict, warn=True)
- info_dict['_filename'] = full_filename = self.prepare_filepath(filename)
- temp_filename = self.prepare_filepath(filename, 'temp')
+ info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True)
+ temp_filename = self.prepare_filename(info_dict, 'temp')
files_to_move = {}
+ skip_dl = self.params.get('skip_download', False)
# Forced printings
self.__forced_printings(info_dict, full_filename, incomplete=False)
# Do nothing else if in simulate mode
return
- if filename is None:
+ if full_filename is None:
return
def ensure_dir_exists(path):
return
if self.params.get('writedescription', False):
- descfn = replace_extension(
- self.prepare_filepath(filename, 'description'),
- 'description', info_dict.get('ext'))
+ descfn = self.prepare_filename(info_dict, 'description')
if not ensure_dir_exists(encodeFilename(descfn)):
return
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
return
if self.params.get('writeannotations', False):
- annofn = replace_extension(
- self.prepare_filepath(filename, 'annotation'),
- 'annotations.xml', info_dict.get('ext'))
+ annofn = self.prepare_filename(info_dict, 'annotation')
if not ensure_dir_exists(encodeFilename(annofn)):
return
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)):
# ie = self.get_info_extractor(info_dict['extractor_key'])
for sub_lang, sub_info in subtitles.items():
sub_format = sub_info['ext']
- sub_filename = subtitles_filename(temp_filename, sub_lang, sub_format, info_dict.get('ext'))
- sub_filename_final = subtitles_filename(
- self.prepare_filepath(filename, 'subtitle'),
+ sub_fn = self.prepare_filename(info_dict, 'subtitle')
+ sub_filename = subtitles_filename(
+ temp_filename if not skip_dl else sub_fn,
sub_lang, sub_format, info_dict.get('ext'))
+ sub_filename_final = subtitles_filename(sub_fn, sub_lang, sub_format, info_dict.get('ext'))
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
files_to_move[sub_filename] = sub_filename_final
(sub_lang, error_to_compat_str(err)))
continue
- if self.params.get('skip_download', False):
+ if skip_dl:
if self.params.get('convertsubtitles', False):
# subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles'))
- filename_real_ext = os.path.splitext(filename)[1][1:]
+ filename_real_ext = os.path.splitext(full_filename)[1][1:]
filename_wo_ext = (
os.path.splitext(full_filename)[0]
if filename_real_ext == info_dict['ext']
else:
try:
self.post_process(full_filename, info_dict, files_to_move)
- except (PostProcessingError) as err:
- self.report_error('postprocessing: %s' % str(err))
+ except PostProcessingError as err:
+ self.report_error('Postprocessing: %s' % str(err))
return
if self.params.get('writeinfojson', False):
- infofn = replace_extension(
- self.prepare_filepath(filename, 'infojson'),
- 'info.json', info_dict.get('ext'))
+ infofn = self.prepare_filename(info_dict, 'infojson')
if not ensure_dir_exists(encodeFilename(infofn)):
return
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
- self.to_screen('[info] Video description metadata is already present')
+ self.to_screen('[info] Video metadata is already present')
else:
- self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
+ self.to_screen('[info] Writing video metadata as JSON to: ' + infofn)
try:
write_json_file(self.filter_requested_info(info_dict), infofn)
except (OSError, IOError):
- self.report_error('Cannot write metadata to JSON file ' + infofn)
+ self.report_error('Cannot write video metadata to JSON file ' + infofn)
return
- info_dict['__infojson_filepath'] = infofn
+ info_dict['__infojson_filename'] = infofn
- thumbdir = os.path.dirname(self.prepare_filepath(filename, 'thumbnail'))
- for thumbfn in self._write_thumbnails(info_dict, temp_filename):
- files_to_move[thumbfn] = os.path.join(thumbdir, os.path.basename(thumbfn))
+ thumbfn = self.prepare_filename(info_dict, 'thumbnail')
+ thumb_fn_temp = temp_filename if not skip_dl else thumbfn
+ for thumb_ext in self._write_thumbnails(info_dict, thumb_fn_temp):
+ thumb_filename_temp = replace_extension(thumb_fn_temp, thumb_ext, info_dict.get('ext'))
+ thumb_filename = replace_extension(thumbfn, thumb_ext, info_dict.get('ext'))
+ files_to_move[thumb_filename_temp] = info_dict['__thumbnail_filename'] = thumb_filename
# Write internet shortcut files
url_link = webloc_link = desktop_link = False
# Download
must_record_download_archive = False
- if not self.params.get('skip_download', False):
+ if not skip_dl:
try:
- def existing_file(filename, temp_filename):
- file_exists = os.path.exists(encodeFilename(filename))
- tempfile_exists = (
- False if temp_filename == filename
- else os.path.exists(encodeFilename(temp_filename)))
- if not self.params.get('overwrites', False) and (file_exists or tempfile_exists):
- existing_filename = temp_filename if tempfile_exists else filename
- self.to_screen('[download] %s has already been downloaded and merged' % existing_filename)
- return existing_filename
- if tempfile_exists:
- self.report_file_delete(temp_filename)
- os.remove(encodeFilename(temp_filename))
- if file_exists:
- self.report_file_delete(filename)
- os.remove(encodeFilename(filename))
- return None
+ def existing_file(*filepaths):
+ ext = info_dict.get('ext')
+ final_ext = self.params.get('final_ext', ext)
+ existing_files = []
+ for file in orderedSet(filepaths):
+ if final_ext != ext:
+ converted = replace_extension(file, final_ext, ext)
+ if os.path.exists(encodeFilename(converted)):
+ existing_files.append(converted)
+ if os.path.exists(encodeFilename(file)):
+ existing_files.append(file)
+
+ if not existing_files or self.params.get('overwrites', False):
+ for file in orderedSet(existing_files):
+ self.report_file_delete(file)
+ os.remove(encodeFilename(file))
+ return None
+
+ self.report_file_already_downloaded(existing_files[0])
+ info_dict['ext'] = os.path.splitext(existing_files[0])[1][1:]
+ return existing_files[0]
success = True
if info_dict.get('requested_formats') is not None:
downloaded = []
merger = FFmpegMergerPP(self)
- if not merger.available:
- postprocessors = []
- self.report_warning('You have requested multiple '
- 'formats but ffmpeg is not installed.'
- ' The formats won\'t be merged.')
- else:
- postprocessors = [merger]
+ if self.params.get('allow_unplayable_formats'):
+ self.report_warning(
+ 'You have requested merging of multiple formats '
+ 'while also allowing unplayable formats to be downloaded. '
+ 'The formats won\'t be merged to prevent data corruption.')
+ elif not merger.available:
+ self.report_warning(
+ 'You have requested merging of multiple formats but ffmpeg is not installed. '
+ 'The formats won\'t be merged.')
def compatible_formats(formats):
# TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
full_filename = correct_ext(full_filename)
temp_filename = correct_ext(temp_filename)
dl_filename = existing_file(full_filename, temp_filename)
+ info_dict['__real_download'] = False
if dl_filename is None:
for f in requested_formats:
new_info = dict(info_dict)
new_info.update(f)
fname = prepend_extension(
- self.prepare_filepath(self.prepare_filename(new_info), 'temp'),
+ self.prepare_filename(new_info, 'temp'),
'f%s' % f['format_id'], new_info['ext'])
if not ensure_dir_exists(fname):
return
downloaded.append(fname)
partial_success, real_download = dl(fname, new_info)
+ info_dict['__real_download'] = info_dict['__real_download'] or real_download
success = success and partial_success
- info_dict['__postprocessors'] = postprocessors
- info_dict['__files_to_merge'] = downloaded
- # Even if there were no downloads, it is being merged only now
- info_dict['__real_download'] = True
+ if merger.available and not self.params.get('allow_unplayable_formats'):
+ info_dict['__postprocessors'].append(merger)
+ info_dict['__files_to_merge'] = downloaded
+ # Even if there were no downloads, it is being merged only now
+ info_dict['__real_download'] = True
+ else:
+ for file in downloaded:
+ files_to_move[file] = None
else:
# Just a single file
dl_filename = existing_file(full_filename, temp_filename)
self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
return
- if success and filename != '-':
+ if success and full_filename != '-':
# Fixup content
fixup_policy = self.params.get('fixup')
if fixup_policy is None:
assert fixup_policy in ('ignore', 'never')
if (info_dict.get('requested_formats') is None
- and info_dict.get('container') == 'm4a_dash'):
+ and info_dict.get('container') == 'm4a_dash'
+ and info_dict.get('ext') == 'm4a'):
if fixup_policy == 'warn':
self.report_warning(
'%s: writing DASH m4a. '
try:
self.post_process(dl_filename, info_dict, files_to_move)
- except (PostProcessingError) as err:
- self.report_error('postprocessing: %s' % str(err))
+ except PostProcessingError as err:
+ self.report_error('Postprocessing: %s' % str(err))
return
try:
for ph in self._post_hooks:
def download(self, url_list):
"""Download a given list of URLs."""
- outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
+ outtmpl = self.outtmpl_dict['default']
if (len(url_list) > 1
and outtmpl != '-'
and '%' not in outtmpl
@staticmethod
def filter_requested_info(info_dict):
+ fields_to_remove = ('requested_formats', 'requested_subtitles')
return dict(
(k, v) for k, v in info_dict.items()
- if k not in ['requested_formats', 'requested_subtitles'])
+ if (k[0] != '_' or k == '_type') and k not in fields_to_remove)
def run_pp(self, pp, infodict, files_to_move={}):
files_to_delete = []
- try:
- files_to_delete, infodict = pp.run(infodict)
- except PostProcessingError as e:
- self.report_error(e.msg)
+ files_to_delete, infodict = pp.run(infodict)
if not files_to_delete:
return files_to_move, infodict
"""Run all the postprocessors on the given file."""
info = dict(ie_info)
info['filepath'] = filename
+ info['__files_to_move'] = {}
for pp in ie_info.get('__postprocessors', []) + self._pps['normal']:
files_to_move, info = self.run_pp(pp, info, files_to_move)
- info = self.run_pp(MoveFilesAfterDownloadPP(self, files_to_move), info, files_to_move)[1]
+ info = self.run_pp(MoveFilesAfterDownloadPP(self, files_to_move), info)[1]
for pp in self._pps['aftermove']:
- files_to_move, info = self.run_pp(pp, info, {})
+ info = self.run_pp(pp, info, {})[1]
def _make_archive_id(self, info_dict):
video_id = info_dict.get('id')
'|',
format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes),
format_field(f, 'tbr', '%4dk'),
- f.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n"),
+ f.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n").replace('niconico_', ''),
'|',
format_field(f, 'vcodec', default='unknown').replace('none', ''),
format_field(f, 'vbr', '%4dk'),
if f.get('preference') is None or f['preference'] >= -1000]
header_line = ['format code', 'extension', 'resolution', 'note']
- # if len(formats) > 1:
- # table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
self.to_screen(
'[info] Available formats for %s:\n%s' % (info_dict['id'], render_table(
header_line,
self.get_encoding()))
write_string(encoding_str, encoding=None)
- self._write_string('[debug] yt-dlp version %s\n' % __version__)
+ source = (
+ '(exe)' if hasattr(sys, 'frozen')
+ else '(zip)' if isinstance(globals().get('__loader__'), zipimporter)
+ else '(source)' if os.path.basename(sys.argv[0]) == '__main__.py'
+ else '')
+ self._write_string('[debug] yt-dlp version %s %s\n' % (__version__, source))
if _LAZY_LOADER:
self._write_string('[debug] Lazy loading extractors enabled\n')
if _PLUGIN_CLASSES:
return impl_name + ' version %d.%d.%d' % sys.pypy_version_info[:3]
return impl_name
- self._write_string('[debug] Python version %s (%s) - %s\n' % (
- platform.python_version(), python_implementation(),
+ self._write_string('[debug] Python version %s (%s %s) - %s\n' % (
+ platform.python_version(),
+ python_implementation(),
+ platform.architecture()[0],
platform_name()))
exe_versions = FFmpegPostProcessor.get_versions(self)
encoding = preferredencoding()
return encoding
- def _write_thumbnails(self, info_dict, filename):
- if self.params.get('writethumbnail', False):
- thumbnails = info_dict.get('thumbnails')
- if thumbnails:
- thumbnails = [thumbnails[-1]]
- elif self.params.get('write_all_thumbnails', False):
+ def _write_thumbnails(self, info_dict, filename): # return the extensions
+ write_all = self.params.get('write_all_thumbnails', False)
+ thumbnails = []
+ if write_all or self.params.get('writethumbnail', False):
thumbnails = info_dict.get('thumbnails') or []
- else:
- thumbnails = []
+ multiple = write_all and len(thumbnails) > 1
ret = []
- for t in thumbnails:
+ for t in thumbnails[::1 if write_all else -1]:
thumb_ext = determine_ext(t['url'], 'jpg')
- suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
- thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
- t['filename'] = thumb_filename = replace_extension(filename + suffix, thumb_ext, info_dict.get('ext'))
+ suffix = '%s.' % t['id'] if multiple else ''
+ thumb_display_id = '%s ' % t['id'] if multiple else ''
+ t['filename'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext'))
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)):
- ret.append(thumb_filename)
+ ret.append(suffix + thumb_ext)
self.to_screen('[%s] %s: Thumbnail %sis already present' %
(info_dict['extractor'], info_dict['id'], thumb_display_id))
else:
uf = self.urlopen(t['url'])
with open(encodeFilename(thumb_filename), 'wb') as thumbf:
shutil.copyfileobj(uf, thumbf)
- ret.append(thumb_filename)
+ ret.append(suffix + thumb_ext)
self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
(info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.report_warning('Unable to download thumbnail "%s": %s' %
(t['url'], error_to_compat_str(err)))
+ if ret and not write_all:
+ break
return ret
projects / yt-dlp.git / blobdiff