from .compat import (
compat_basestring,
- compat_cookiejar,
compat_get_terminal_size,
compat_kwargs,
compat_numeric_types,
compat_os_name,
+ compat_shlex_quote,
compat_str,
compat_tokenize_tokenize,
compat_urllib_error,
compat_urllib_request,
compat_urllib_request_DataHandler,
)
+from .cookies import load_cookies
from .utils import (
age_restricted,
args_to_str,
float_or_none,
format_bytes,
format_field,
- STR_FORMAT_RE,
+ STR_FORMAT_RE_TMPL,
+ STR_FORMAT_TYPES,
formatSeconds,
GeoRestrictedError,
HEADRequest,
str_or_none,
strftime_or_none,
subtitles_filename,
+ ThrottledDownload,
to_high_limit_path,
traverse_obj,
+ try_get,
UnavailableVideoError,
url_basename,
+ variadic,
version_tuple,
write_json_file,
write_string,
- YoutubeDLCookieJar,
YoutubeDLCookieProcessor,
YoutubeDLHandler,
YoutubeDLRedirectHandler,
)
from .extractor.openload import PhantomJSwrapper
from .downloader import (
+ FFmpegFD,
get_suitable_downloader,
shorten_protocol_name
)
into a single file
allow_multiple_audio_streams: Allow multiple audio streams to be merged
into a single file
+ check_formats Whether to test if the formats are downloadable.
+ Can be True (check all), False (check none)
+ or None (check only if requested by extractor)
paths: Dictionary of output paths. The allowed keys are 'home'
'temp' and the keys of OUTTMPL_TYPES (in utils.py)
outtmpl: Dictionary of templates for output names. Allowed keys
writedesktoplink: Write a Linux internet shortcut file (.desktop)
writesubtitles: Write the video subtitles to a file
writeautomaticsub: Write the automatically generated subtitles to a file
- allsubtitles: Deprecated - Use subtitlelangs = ['all']
+ allsubtitles: Deprecated - Use subtitleslangs = ['all']
Downloads all the subtitles of the video
(requires writesubtitles or writeautomaticsub)
listsubtitles: Lists all available subtitles for the video
break_on_reject: Stop the download process when encountering a video that
has been filtered out.
cookiefile: File name where cookies should be read from and dumped to
+ cookiesfrombrowser: A tuple containing the name of the browser and the profile
+ name/path from where cookies are loaded.
+ Eg: ('chrome', ) or (vivaldi, 'default')
nocheckcertificate:Do not verify SSL certificates
prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
At the moment, this is only supported by YouTube.
progress, with a dictionary with the entries
* status: One of "downloading", "error", or "finished".
Check this first and ignore unknown values.
+ * info_dict: The extracted info_dict
If status is one of "downloading", or "finished", the
following properties may also be present:
if True, otherwise use ffmpeg/avconv if False, otherwise
use downloader suggested by extractor if None.
compat_opts: Compatibility options. See "Differences in default behavior".
- Note that only format-sort, format-spec, no-live-chat,
- no-attach-info-json, playlist-index, list-formats,
- no-direct-merge, embed-thumbnail-atomicparsley,
- no-youtube-unavailable-videos, no-youtube-channel-redirect,
- works when used via the API
+ The following options do not work when used through the API:
+ filename, abort-on-error, multistreams, no-live-chat,
+ no-clean-infojson, no-playlist-metafiles, no-keep-subs.
+ Refer __init__.py for their implementation
The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see yt_dlp/downloader/common.py):
- nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
- noresizebuffer, retries, continuedl, noprogress, consoletitle,
- xattr_set_filesize, external_downloader_args, hls_use_mpegts,
- http_chunk_size.
+ nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
+ max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle,
+ xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size.
The following options are used by the post processors:
prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available,
dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
hls_split_discontinuity: Split HLS playlists to different formats at
discontinuities such as ad breaks (default: False)
- youtube_include_dash_manifest: If True (default), DASH manifests and related
+ extractor_args: A dictionary of arguments to be passed to the extractors.
+ See "EXTRACTOR ARGUMENTS" for details.
+ Eg: {'youtube': {'skip': ['dash', 'hls']}}
+ youtube_include_dash_manifest: Deprecated - Use extractor_args instead.
+ If True (default), DASH manifests and related
data will be downloaded and processed by extractor.
You can reduce network I/O by disabling it if you don't
care about DASH. (only for youtube)
- youtube_include_hls_manifest: If True (default), HLS manifests and related
+ youtube_include_hls_manifest: Deprecated - Use extractor_args instead.
+ If True (default), HLS manifests and related
data will be downloaded and processed by extractor.
You can reduce network I/O by disabling it if you don't
care about HLS. (only for youtube)
params = None
_ies = []
_pps = {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
- __prepare_filename_warned = False
+ _printed_messages = set()
_first_webpage_request = True
_download_retcode = None
_num_downloads = None
self._ies = []
self._ies_instances = {}
self._pps = {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
- self.__prepare_filename_warned = False
+ self._printed_messages = set()
self._first_webpage_request = True
self._post_hooks = []
self._progress_hooks = []
for _ in range(line_count))
return res[:-len('\n')]
- def _write_string(self, s, out=None):
- write_string(s, out=out, encoding=self.params.get('encoding'))
+ def _write_string(self, message, out=None, only_once=False):
+ if only_once:
+ if message in self._printed_messages:
+ return
+ self._printed_messages.add(message)
+ write_string(message, out=out, encoding=self.params.get('encoding'))
def to_stdout(self, message, skip_eol=False, quiet=False):
"""Print message to stdout"""
'%s%s' % (self._bidi_workaround(message), ('' if skip_eol else '\n')),
self._err_file if quiet else self._screen_file)
- def to_stderr(self, message):
+ def to_stderr(self, message, only_once=False):
"""Print message to stderr"""
assert isinstance(message, compat_str)
if self.params.get('logger'):
self.params['logger'].error(message)
else:
- self._write_string('%s\n' % self._bidi_workaround(message), self._err_file)
+ self._write_string('%s\n' % self._bidi_workaround(message), self._err_file, only_once=only_once)
def to_console_title(self, message):
if not self.params.get('consoletitle', False):
self.to_stdout(
message, skip_eol, quiet=self.params.get('quiet', False))
- def report_warning(self, message):
+ def report_warning(self, message, only_once=False):
'''
Print the message to stderr, it will be prefixed with 'WARNING:'
If stderr is a tty file the 'WARNING:' will be colored
else:
_msg_header = 'WARNING:'
warning_message = '%s %s' % (_msg_header, message)
- self.to_stderr(warning_message)
+ self.to_stderr(warning_message, only_once)
def report_error(self, message, tb=None):
'''
error_message = '%s %s' % (_msg_header, message)
self.trouble(error_message, tb)
- def write_debug(self, message):
+ def write_debug(self, message, only_once=False):
'''Log debug message or Print message to stderr'''
if not self.params.get('verbose', False):
return
if self.params.get('logger'):
self.params['logger'].debug(message)
else:
- self._write_string('%s\n' % message)
+ self.to_stderr(message, only_once)
def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded."""
return sanitize_path(path, force=self.params.get('windowsfilenames'))
@staticmethod
- def validate_outtmpl(tmpl):
+ def _outtmpl_expandpath(outtmpl):
+ # expand_path translates '%%' into '%' and '$$' into '$'
+ # correspondingly that is not what we want since we need to keep
+ # '%%' intact for template dict substitution step. Working around
+ # with boundary-alike separator hack.
+ sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
+ outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
+
+ # outtmpl should be expand_path'ed before template dict substitution
+ # because meta fields may contain env variables we don't want to
+ # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
+ # title "Hello $PATH", we don't want `$PATH` to be expanded.
+ return expand_path(outtmpl).replace(sep, '')
+
+ @staticmethod
+ def escape_outtmpl(outtmpl):
+ ''' Escape any remaining strings like %s, %abc% etc. '''
+ return re.sub(
+ STR_FORMAT_RE_TMPL.format('', '(?![%(\0])'),
+ lambda mobj: ('' if mobj.group('has_key') else '%') + mobj.group(0),
+ outtmpl)
+
+ @classmethod
+ def validate_outtmpl(cls, outtmpl):
''' @return None or Exception object '''
+ outtmpl = re.sub(
+ STR_FORMAT_RE_TMPL.format('[^)]*', '[ljq]'),
+ lambda mobj: f'{mobj.group(0)[:-1]}s',
+ cls._outtmpl_expandpath(outtmpl))
try:
- re.sub(
- STR_FORMAT_RE.format(''),
- lambda mobj: ('%' if not mobj.group('has_key') else '') + mobj.group(0),
- tmpl
- ) % collections.defaultdict(int)
+ cls.escape_outtmpl(outtmpl) % collections.defaultdict(int)
return None
except ValueError as err:
return err
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
- """ Make the template and info_dict suitable for substitution (outtmpl % info_dict)"""
- info_dict = dict(info_dict)
+ """ Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """
+ info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
na = self.params.get('outtmpl_na_placeholder', 'NA')
+ info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList
+ for key in ('__original_infodict', '__postprocessors'):
+ info_dict.pop(key, None)
info_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
formatSeconds(info_dict['duration'], '-' if sanitize else ':')
if info_dict.get('duration', None) is not None
else None)
- info_dict['epoch'] = int(time.time())
info_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
if info_dict.get('resolution') is None:
info_dict['resolution'] = self.format_resolution(info_dict, default=None)
}
TMPL_DICT = {}
- EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE.format('[^)]*'))
+ EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljq]'))
MATH_FUNCTIONS = {
'+': float.__add__,
'-': float.__sub__,
return value
+ def _dumpjson_default(obj):
+ if isinstance(obj, (set, LazyList)):
+ return list(obj)
+ raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
+
def create_key(outer_mobj):
if not outer_mobj.group('has_key'):
- return '%{}'.format(outer_mobj.group(0))
+ return f'%{outer_mobj.group(0)}'
+ prefix = outer_mobj.group('prefix')
key = outer_mobj.group('key')
- fmt = outer_mobj.group('format')
+ original_fmt = fmt = outer_mobj.group('format')
mobj = re.match(INTERNAL_FORMAT_RE, key)
if mobj is None:
value, default, mobj = None, na, {'fields': ''}
value = default if value is None else value
- if fmt == 'c':
- value = compat_str(value)
+ str_fmt = f'{fmt[:-1]}s'
+ if fmt[-1] == 'l':
+ value, fmt = ', '.join(variadic(value)), str_fmt
+ elif fmt[-1] == 'j':
+ value, fmt = json.dumps(value, default=_dumpjson_default), str_fmt
+ elif fmt[-1] == 'q':
+ value, fmt = compat_shlex_quote(str(value)), str_fmt
+ elif fmt[-1] == 'c':
+ value = str(value)
if value is None:
value, fmt = default, 's'
else:
value = float_or_none(value)
if value is None:
value, fmt = default, 's'
+
if sanitize:
if fmt[-1] == 'r':
# If value is an object, sanitize might convert it to a string
# So we convert it to repr first
- value, fmt = repr(value), '%ss' % fmt[:-1]
+ value, fmt = repr(value), str_fmt
if fmt[-1] in 'csr':
value = sanitize(mobj['fields'].split('.')[-1], value)
- key += '\0%s' % fmt
+
+ key = '%s\0%s' % (key.replace('%', '%\0'), original_fmt)
TMPL_DICT[key] = value
- return '%({key}){fmt}'.format(key=key, fmt=fmt)
+ return f'{prefix}%({key}){fmt}'
return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT
is_id=(k == 'id' or k.endswith('_id')))
outtmpl = self.outtmpl_dict.get(tmpl_type, self.outtmpl_dict['default'])
outtmpl, template_dict = self.prepare_outtmpl(outtmpl, info_dict, sanitize)
-
- # expand_path translates '%%' into '%' and '$$' into '$'
- # correspondingly that is not what we want since we need to keep
- # '%%' intact for template dict substitution step. Working around
- # with boundary-alike separator hack.
- sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
- outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
-
- # outtmpl should be expand_path'ed before template dict substitution
- # because meta fields may contain env variables we don't want to
- # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
- # title "Hello $PATH", we don't want `$PATH` to be expanded.
- filename = expand_path(outtmpl).replace(sep, '') % template_dict
+ outtmpl = self.escape_outtmpl(self._outtmpl_expandpath(outtmpl))
+ filename = outtmpl % template_dict
force_ext = OUTTMPL_TYPES.get(tmpl_type)
if force_ext is not None:
filename = self._prepare_filename(info_dict, dir_type or 'default')
- if warn and not self.__prepare_filename_warned:
+ if warn:
if not self.params.get('paths'):
pass
elif filename == '-':
- self.report_warning('--paths is ignored when an outputting to stdout')
+ self.report_warning('--paths is ignored when an outputting to stdout', only_once=True)
elif os.path.isabs(filename):
- self.report_warning('--paths is ignored since an absolute path is given in output template')
+ self.report_warning('--paths is ignored since an absolute path is given in output template', only_once=True)
self.__prepare_filename_warned = True
if filename == '-' or not filename:
return filename
else:
self.report_error('no suitable InfoExtractor for URL %s' % url)
- def __handle_extraction_exceptions(func):
+ def __handle_extraction_exceptions(func, handle_all_errors=True):
def wrapper(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
self.report_error(msg)
except ExtractorError as e: # An error we somewhat expected
self.report_error(compat_str(e), e.format_traceback())
+ except ThrottledDownload:
+ self.to_stderr('\r')
+ self.report_warning('The download speed is below throttle limit. Re-extracting data')
+ return wrapper(self, *args, **kwargs)
except (MaxDownloadsReached, ExistingVideoReached, RejectedVideoReached):
raise
except Exception as e:
- if self.params.get('ignoreerrors', False):
+ if handle_all_errors and self.params.get('ignoreerrors', False):
self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
else:
raise
'_type': 'compat_list',
'entries': ie_result,
}
+ if extra_info.get('original_url'):
+ ie_result.setdefault('original_url', extra_info['original_url'])
self.add_default_extra_info(ie_result, ie, url)
if process:
return self.process_ie_result(ie_result, download, extra_info)
return ie_result
def add_default_extra_info(self, ie_result, ie, url):
- self.add_extra_info(ie_result, {
- 'extractor': ie.IE_NAME,
- 'webpage_url': url,
- 'original_url': url,
- 'webpage_url_basename': url_basename(url),
- 'extractor_key': ie.ie_key(),
- })
+ if url is not None:
+ self.add_extra_info(ie_result, {
+ 'webpage_url': url,
+ 'original_url': url,
+ 'webpage_url_basename': url_basename(url),
+ })
+ if ie is not None:
+ self.add_extra_info(ie_result, {
+ 'extractor': ie.IE_NAME,
+ 'extractor_key': ie.ie_key(),
+ })
def process_ie_result(self, ie_result, download=True, extra_info={}):
"""
if result_type in ('url', 'url_transparent'):
ie_result['url'] = sanitize_url(ie_result['url'])
+ if ie_result.get('original_url'):
+ extra_info.setdefault('original_url', ie_result['original_url'])
+
extract_flat = self.params.get('extract_flat', False)
if ((extract_flat == 'in_playlist' and 'playlist' in extra_info)
or extract_flat is True):
info_copy = ie_result.copy()
self.add_extra_info(info_copy, extra_info)
- self.add_default_extra_info(
- info_copy, self.get_info_extractor(ie_result.get('ie_key')), ie_result['url'])
+ ie = try_get(ie_result.get('ie_key'), self.get_info_extractor)
+ self.add_default_extra_info(info_copy, ie, ie_result['url'])
self.__forced_printings(info_copy, self.prepare_filename(info_copy), incomplete=True)
return ie_result
if not isinstance(ie_entries, (list, PagedList)):
ie_entries = LazyList(ie_entries)
+ def get_entry(i):
+ return YoutubeDL.__handle_extraction_exceptions(
+ lambda self, i: ie_entries[i - 1],
+ False
+ )(self, i)
+
entries = []
for i in playlistitems or itertools.count(playliststart):
if playlistitems is None and playlistend is not None and playlistend < i:
break
entry = None
try:
- entry = ie_entries[i - 1]
+ entry = get_entry(i)
if entry is None:
raise EntryNotInPlaylist()
except (IndexError, EntryNotInPlaylist):
else:
self.to_screen('[info] Writing playlist metadata as JSON to: ' + infofn)
try:
- write_json_file(self.filter_requested_info(ie_result, self.params.get('clean_infojson', True)), infofn)
+ write_json_file(self.sanitize_info(ie_result, self.params.get('clean_infojson', True)), infofn)
except (OSError, IOError):
self.report_error('Cannot write playlist metadata to JSON file ' + infofn)
if not allow_multiple_streams[aud_vid] and fmt_info.get(aud_vid[0] + 'codec') != 'none':
if get_no_more[aud_vid]:
formats_info.pop(i)
+ break
get_no_more[aud_vid] = True
if len(formats_info) == 1:
return new_dict
def _check_formats(formats):
+ if not check_formats:
+ yield from formats
+ return
for f in formats:
self.to_screen('[info] Testing format %s' % f['format_id'])
temp_file = tempfile.NamedTemporaryFile(
dir=self.get_output_path('temp') or None)
temp_file.close()
try:
- dl, _ = self.dl(temp_file.name, f, test=True)
- except (ExtractorError, IOError, OSError, ValueError) + network_exceptions:
- dl = False
+ success, _ = self.dl(temp_file.name, f, test=True)
+ except (DownloadError, IOError, OSError, ValueError) + network_exceptions:
+ success = False
finally:
if os.path.exists(temp_file.name):
try:
os.remove(temp_file.name)
except OSError:
self.report_warning('Unable to delete temporary file "%s"' % temp_file.name)
- if dl:
+ if success:
yield f
else:
self.to_screen('[info] Unable to download format %s. Skipping...' % f['format_id'])
def selector_function(ctx):
for f in fs:
- for format in f(ctx):
- yield format
+ yield from f(ctx)
return selector_function
elif selector.type == GROUP: # ()
return picked_formats
return []
+ elif selector.type == MERGE: # +
+ selector_1, selector_2 = map(_build_selector_function, selector.selector)
+
+ def selector_function(ctx):
+ for pair in itertools.product(
+ selector_1(copy.deepcopy(ctx)), selector_2(copy.deepcopy(ctx))):
+ yield _merge(pair)
+
elif selector.type == SINGLE: # atom
format_spec = selector.selector or 'best'
# TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
if format_spec == 'all':
def selector_function(ctx):
- formats = list(ctx['formats'])
- if check_formats:
- formats = _check_formats(formats)
- for f in formats:
- yield f
+ yield from _check_formats(ctx['formats'])
elif format_spec == 'mergeall':
def selector_function(ctx):
- formats = ctx['formats']
- if check_formats:
- formats = list(_check_formats(formats))
+ formats = list(_check_formats(ctx['formats']))
if not formats:
return
merged_format = formats[-1]
def selector_function(ctx):
formats = list(ctx['formats'])
- if not formats:
- return
matches = list(filter(filter_f, formats)) if filter_f is not None else formats
if format_fallback and ctx['incomplete_formats'] and not matches:
# for extractors with incomplete formats (audio only (soundcloud)
# or video only (imgur)) best/worst will fallback to
# best/worst {video,audio}-only format
matches = formats
- if format_reverse:
- matches = matches[::-1]
- if check_formats:
- matches = list(itertools.islice(_check_formats(matches), format_idx))
- n = len(matches)
- if -n <= format_idx - 1 < n:
+ matches = LazyList(_check_formats(matches[::-1 if format_reverse else 1]))
+ try:
yield matches[format_idx - 1]
-
- elif selector.type == MERGE: # +
- selector_1, selector_2 = map(_build_selector_function, selector.selector)
-
- def selector_function(ctx):
- for pair in itertools.product(
- selector_1(copy.deepcopy(ctx)), selector_2(copy.deepcopy(ctx))):
- yield _merge(pair)
+ except IndexError:
+ return
filters = [self._build_format_filter(f) for f in selector.filters]
t.get('id') if t.get('id') is not None else '',
t.get('url')))
- def test_thumbnail(t):
- self.to_screen('[info] Testing thumbnail %s' % t['id'])
- try:
- self.urlopen(HEADRequest(t['url']))
- except network_exceptions as err:
- self.to_screen('[info] Unable to connect to thumbnail %s URL "%s" - %s. Skipping...' % (
- t['id'], t['url'], error_to_compat_str(err)))
- return False
- return True
+ def thumbnail_tester():
+ if self.params.get('check_formats'):
+ test_all = True
+ to_screen = lambda msg: self.to_screen(f'[info] {msg}')
+ else:
+ test_all = False
+ to_screen = self.write_debug
+
+ def test_thumbnail(t):
+ if not test_all and not t.get('_test_url'):
+ return True
+ to_screen('Testing thumbnail %s' % t['id'])
+ try:
+ self.urlopen(HEADRequest(t['url']))
+ except network_exceptions as err:
+ to_screen('Unable to connect to thumbnail %s URL "%s" - %s. Skipping...' % (
+ t['id'], t['url'], error_to_compat_str(err)))
+ return False
+ return True
+
+ return test_thumbnail
for i, t in enumerate(thumbnails):
if t.get('id') is None:
if t.get('width') and t.get('height'):
t['resolution'] = '%dx%d' % (t['width'], t['height'])
t['url'] = sanitize_url(t['url'])
- if self.params.get('check_formats'):
- info_dict['thumbnails'] = reversed(LazyList(filter(test_thumbnail, thumbnails[::-1])))
+
+ if self.params.get('check_formats') is not False:
+ info_dict['thumbnails'] = LazyList(filter(thumbnail_tester(), thumbnails[::-1])).reverse()
+ else:
+ info_dict['thumbnails'] = thumbnails
def process_video_result(self, info_dict, download=True):
assert info_dict.get('_type', 'video') == 'video'
self._sanitize_thumbnails(info_dict)
- if self.params.get('list_thumbnails'):
- self.list_thumbnails(info_dict)
- return
-
thumbnail = info_dict.get('thumbnail')
thumbnails = info_dict.get('thumbnails')
if thumbnail:
elif thumbnails:
info_dict['thumbnail'] = thumbnails[-1]['url']
- if 'display_id' not in info_dict and 'id' in info_dict:
+ if info_dict.get('display_id') is None and 'id' in info_dict:
info_dict['display_id'] = info_dict['id']
for ts_key, date_key in (
except (ValueError, OverflowError, OSError):
pass
+ live_keys = ('is_live', 'was_live')
+ live_status = info_dict.get('live_status')
+ if live_status is None:
+ for key in live_keys:
+ if info_dict.get(key) is False:
+ continue
+ if info_dict.get(key):
+ live_status = key
+ break
+ if all(info_dict.get(key) is False for key in live_keys):
+ live_status = 'not_live'
+ if live_status:
+ info_dict['live_status'] = live_status
+ for key in live_keys:
+ if info_dict.get(key) is None:
+ info_dict[key] = (live_status == key)
+
# Auto generate title fields corresponding to the *_number fields when missing
# in order to always have clean titles. This is very common for TV series.
for field in ('chapter', 'season', 'episode'):
automatic_captions = info_dict.get('automatic_captions')
subtitles = info_dict.get('subtitles')
- if self.params.get('listsubtitles', False):
- if 'automatic_captions' in info_dict:
- self.list_subtitles(
- info_dict['id'], automatic_captions, 'automatic captions')
- self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
- return
-
info_dict['requested_subtitles'] = self.process_subtitles(
info_dict['id'], subtitles, automatic_captions)
info_dict, _ = self.pre_process(info_dict)
- if self.params.get('listformats'):
- if not info_dict.get('formats'):
- raise ExtractorError('No video formats found', expected=True)
- self.list_formats(info_dict)
+ list_only = self.params.get('list_thumbnails') or self.params.get('listformats') or self.params.get('listsubtitles')
+ if list_only:
+ self.__forced_printings(info_dict, self.prepare_filename(info_dict), incomplete=True)
+ if self.params.get('list_thumbnails'):
+ self.list_thumbnails(info_dict)
+ if self.params.get('listformats'):
+ if not info_dict.get('formats'):
+ raise ExtractorError('No video formats found', expected=True)
+ self.list_formats(info_dict)
+ if self.params.get('listsubtitles'):
+ if 'automatic_captions' in info_dict:
+ self.list_subtitles(
+ info_dict['id'], automatic_captions, 'automatic captions')
+ self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
return
format_selector = self.format_selector
raise ExtractorError('Requested format is not available', expected=True)
else:
self.report_warning('Requested format is not available')
+ # Process what we can, even without any available formats.
+ self.process_info(dict(info_dict))
elif download:
self.to_screen(
'[info] %s: Downloading %d format(s): %s' % (
if re.match(r'\w+$', tmpl):
tmpl = '%({})s'.format(tmpl)
tmpl, info_copy = self.prepare_outtmpl(tmpl, info_dict)
- self.to_stdout(tmpl % info_copy)
+ self.to_stdout(self.escape_outtmpl(tmpl) % info_copy)
print_mandatory('title')
print_mandatory('id')
if self.params.get('forcejson', False):
self.post_extract(info_dict)
- self.to_stdout(json.dumps(info_dict, default=repr))
+ self.to_stdout(json.dumps(self.sanitize_info(info_dict)))
def dl(self, name, info, subtitle=False, test=False):
}
else:
params = self.params
- fd = get_suitable_downloader(info, params)(self, params)
+ fd = get_suitable_downloader(info, params, to_stdout=(name == '-'))(self, params)
if not test:
for ph in self._progress_hooks:
fd.add_progress_hook(ph)
# TODO: backward compatibility, to be removed
info_dict['fulltitle'] = info_dict['title']
- if 'format' not in info_dict:
+ if 'format' not in info_dict and 'ext' in info_dict:
info_dict['format'] = info_dict['ext']
if self._match_entry(info_dict) is not None:
files_to_move = {}
# Forced printings
- self.__forced_printings(info_dict, full_filename, incomplete=False)
+ self.__forced_printings(info_dict, full_filename, incomplete=('format' not in info_dict))
if self.params.get('simulate', False):
if self.params.get('force_write_download_archive', False):
else:
self.to_screen('[info] Writing video metadata as JSON to: ' + infofn)
try:
- write_json_file(self.filter_requested_info(info_dict, self.params.get('clean_infojson', True)), infofn)
+ write_json_file(self.sanitize_info(info_dict, self.params.get('clean_infojson', True)), infofn)
except (OSError, IOError):
self.report_error('Cannot write video metadata to JSON file ' + infofn)
return
requested_formats = info_dict['requested_formats']
old_ext = info_dict['ext']
- if self.params.get('merge_output_format') is None:
- if not compatible_formats(requested_formats):
- info_dict['ext'] = 'mkv'
- self.report_warning(
- 'Requested formats are incompatible for merge and will be merged into mkv.')
- if (info_dict['ext'] == 'webm'
- and self.params.get('writethumbnail', False)
- and info_dict.get('thumbnails')):
- info_dict['ext'] = 'mkv'
- self.report_warning(
- 'webm doesn\'t support embedding a thumbnail, mkv will be used.')
-
- def correct_ext(filename):
+ if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
+ info_dict['ext'] = 'mkv'
+ self.report_warning(
+ 'Requested formats are incompatible for merge and will be merged into mkv.')
+ new_ext = info_dict['ext']
+
+ def correct_ext(filename, ext=new_ext):
+ if filename == '-':
+ return filename
filename_real_ext = os.path.splitext(filename)[1][1:]
filename_wo_ext = (
os.path.splitext(filename)[0]
- if filename_real_ext == old_ext
+ if filename_real_ext in (old_ext, new_ext)
else filename)
- return '%s.%s' % (filename_wo_ext, info_dict['ext'])
+ return '%s.%s' % (filename_wo_ext, ext)
# Ensure filename always has a correct extension for successful merge
full_filename = correct_ext(full_filename)
info_dict['__real_download'] = False
_protocols = set(determine_protocol(f) for f in requested_formats)
- if len(_protocols) == 1:
+ if len(_protocols) == 1: # All requested formats have same protocol
info_dict['protocol'] = _protocols.pop()
- directly_mergable = (
- 'no-direct-merge' not in self.params.get('compat_opts', [])
- and info_dict.get('protocol') is not None # All requested formats have same protocol
- and not self.params.get('allow_unplayable_formats')
- and get_suitable_downloader(info_dict, self.params).__name__ == 'FFmpegFD')
- if directly_mergable:
- info_dict['url'] = requested_formats[0]['url']
- # Treat it as a single download
- dl_filename = existing_file(full_filename, temp_filename)
- if dl_filename is None:
- success, real_download = self.dl(temp_filename, info_dict)
- info_dict['__real_download'] = real_download
+ directly_mergable = FFmpegFD.can_merge_formats(info_dict)
+ if dl_filename is not None:
+ pass
+ elif (directly_mergable and get_suitable_downloader(
+ info_dict, self.params, to_stdout=(temp_filename == '-')) == FFmpegFD):
+ info_dict['url'] = '\n'.join(f['url'] for f in requested_formats)
+ success, real_download = self.dl(temp_filename, info_dict)
+ info_dict['__real_download'] = real_download
else:
downloaded = []
merger = FFmpegMergerPP(self)
'You have requested merging of multiple formats but ffmpeg is not installed. '
'The formats won\'t be merged.')
- if dl_filename is None:
- for f in requested_formats:
- new_info = dict(info_dict)
- del new_info['requested_formats']
- new_info.update(f)
+ if temp_filename == '-':
+ reason = ('using a downloader other than ffmpeg' if directly_mergable
+ else 'but the formats are incompatible for simultaneous download' if merger.available
+ else 'but ffmpeg is not installed')
+ self.report_warning(
+ f'You have requested downloading multiple formats to stdout {reason}. '
+ 'The formats will be streamed one after the other')
+ fname = temp_filename
+ for f in requested_formats:
+ new_info = dict(info_dict)
+ del new_info['requested_formats']
+ new_info.update(f)
+ if temp_filename != '-':
fname = prepend_extension(
- self.prepare_filename(new_info, 'temp'),
+ correct_ext(temp_filename, new_info['ext']),
'f%s' % f['format_id'], new_info['ext'])
if not self._ensure_dir_exists(fname):
return
downloaded.append(fname)
- partial_success, real_download = self.dl(fname, new_info)
- info_dict['__real_download'] = info_dict['__real_download'] or real_download
- success = success and partial_success
- if merger.available and not self.params.get('allow_unplayable_formats'):
- info_dict['__postprocessors'].append(merger)
- info_dict['__files_to_merge'] = downloaded
- # Even if there were no downloads, it is being merged only now
- info_dict['__real_download'] = True
- else:
- for file in downloaded:
- files_to_move[file] = None
+ partial_success, real_download = self.dl(fname, new_info)
+ info_dict['__real_download'] = info_dict['__real_download'] or real_download
+ success = success and partial_success
+ if merger.available and not self.params.get('allow_unplayable_formats'):
+ info_dict['__postprocessors'].append(merger)
+ info_dict['__files_to_merge'] = downloaded
+ # Even if there were no downloads, it is being merged only now
+ info_dict['__real_download'] = True
+ else:
+ for file in downloaded:
+ files_to_move[file] = None
else:
# Just a single file
dl_filename = existing_file(full_filename, temp_filename)
else:
if self.params.get('dump_single_json', False):
self.post_extract(res)
- self.to_stdout(json.dumps(res, default=repr))
+ self.to_stdout(json.dumps(self.sanitize_info(res)))
return self._download_retcode
[info_filename], mode='r',
openhook=fileinput.hook_encoded('utf-8'))) as f:
# FileInput doesn't have a read method, we can't call json.load
- info = self.filter_requested_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True))
+ info = self.sanitize_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True))
try:
self.process_ie_result(info, download=True)
- except (DownloadError, EntryNotInPlaylist):
+ except (DownloadError, EntryNotInPlaylist, ThrottledDownload):
webpage_url = info.get('webpage_url')
if webpage_url is not None:
self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
return self._download_retcode
@staticmethod
- def filter_requested_info(info_dict, actually_filter=True):
- remove_keys = ['__original_infodict'] # Always remove this since this may contain a copy of the entire dict
+ def sanitize_info(info_dict, remove_private_keys=False):
+ ''' Sanitize the infodict for converting to json '''
+ info_dict.setdefault('epoch', int(time.time()))
+ remove_keys = {'__original_infodict'} # Always remove this since this may contain a copy of the entire dict
keep_keys = ['_type'], # Always keep this to facilitate load-info-json
- if actually_filter:
- remove_keys += ('requested_formats', 'requested_subtitles', 'requested_entries', 'filepath', 'entries', 'original_url')
+ if remove_private_keys:
+ remove_keys |= {
+ 'requested_formats', 'requested_subtitles', 'requested_entries',
+ 'filepath', 'entries', 'original_url', 'playlist_autonumber',
+ }
empty_values = (None, {}, [], set(), tuple())
reject = lambda k, v: k not in keep_keys and (
k.startswith('_') or k in remove_keys or v in empty_values)
else:
- info_dict['epoch'] = int(time.time())
reject = lambda k, v: k in remove_keys
filter_fn = lambda obj: (
list(map(filter_fn, obj)) if isinstance(obj, (LazyList, list, tuple, set))
else dict((k, filter_fn(v)) for k, v in obj.items() if not reject(k, v)))
return filter_fn(info_dict)
+ @staticmethod
+ def filter_requested_info(info_dict, actually_filter=True):
+ ''' Alias of sanitize_info for backward compatibility '''
+ return YoutubeDL.sanitize_info(info_dict, actually_filter)
+
def run_pp(self, pp, infodict):
files_to_delete = []
if '__files_to_move' not in infodict:
res += '~' + format_bytes(fdict['filesize_approx'])
return res
- def _format_note_table(self, f):
- def join_fields(*vargs):
- return ', '.join((val for val in vargs if val != ''))
-
- return join_fields(
- 'UNSUPPORTED' if f.get('ext') in ('f4f', 'f4m') else '',
- format_field(f, 'language', '[%s]'),
- format_field(f, 'format_note'),
- format_field(f, 'container', ignore=(None, f.get('ext'))),
- format_field(f, 'asr', '%5dHz'))
-
def list_formats(self, info_dict):
formats = info_dict.get('formats', [info_dict])
new_format = (
'list-formats' not in self.params.get('compat_opts', [])
- and self.params.get('list_formats_as_table', True) is not False)
+ and self.params.get('listformats_table', True) is not False)
if new_format:
table = [
[
format_field(f, 'acodec', default='unknown').replace('none', ''),
format_field(f, 'abr', '%3dk'),
format_field(f, 'asr', '%5dHz'),
- self._format_note_table(f)]
- for f in formats
- if f.get('preference') is None or f['preference'] >= -1000]
+ ', '.join(filter(None, (
+ 'UNSUPPORTED' if f.get('ext') in ('f4f', 'f4m') else '',
+ format_field(f, 'language', '[%s]'),
+ format_field(f, 'format_note'),
+ format_field(f, 'container', ignore=(None, f.get('ext'))),
+ ))),
+ ] for f in formats if f.get('preference') is None or f['preference'] >= -1000]
header_line = ['ID', 'EXT', 'RESOLUTION', 'FPS', '|', ' FILESIZE', ' TBR', 'PROTO',
- '|', 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'NOTE']
+ '|', 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'MORE INFO']
else:
table = [
[
header_line = ['format code', 'extension', 'resolution', 'note']
self.to_screen(
- '[info] Available formats for %s:\n%s' % (info_dict['id'], render_table(
- header_line,
- table,
- delim=new_format,
- extraGap=(0 if new_format else 1),
- hideEmpty=new_format)))
+ '[info] Available formats for %s:' % info_dict['id'])
+ self.to_stdout(render_table(
+ header_line, table, delim=new_format, extraGap=(0 if new_format else 1), hideEmpty=new_format))
def list_thumbnails(self, info_dict):
thumbnails = list(info_dict.get('thumbnails'))
self.to_screen(
'[info] Thumbnails for %s:' % info_dict['id'])
- self.to_screen(render_table(
+ self.to_stdout(render_table(
['ID', 'width', 'height', 'URL'],
[[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
'Available %s for %s:' % (name, video_id))
def _row(lang, formats):
- exts, names = zip(*((f['ext'], f.get('name', 'unknown')) for f in reversed(formats)))
+ exts, names = zip(*((f['ext'], f.get('name') or 'unknown') for f in reversed(formats)))
if len(set(names)) == 1:
names = [] if names[0] == 'unknown' else names[:1]
return [lang, ', '.join(names), ', '.join(exts)]
- self.to_screen(render_table(
+ self.to_stdout(render_table(
['Language', 'Name', 'Formats'],
[_row(lang, formats) for lang, formats in subtitles.items()],
hideEmpty=True))
timeout_val = self.params.get('socket_timeout')
self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
+ opts_cookiesfrombrowser = self.params.get('cookiesfrombrowser')
opts_cookiefile = self.params.get('cookiefile')
opts_proxy = self.params.get('proxy')
- if opts_cookiefile is None:
- self.cookiejar = compat_cookiejar.CookieJar()
- else:
- opts_cookiefile = expand_path(opts_cookiefile)
- self.cookiejar = YoutubeDLCookieJar(opts_cookiefile)
- if os.access(opts_cookiefile, os.R_OK):
- self.cookiejar.load(ignore_discard=True, ignore_expires=True)
+ self.cookiejar = load_cookies(opts_cookiefile, opts_cookiesfrombrowser, self)
cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
if opts_proxy is not None:
multiple = write_all and len(thumbnails) > 1
ret = []
- for t in thumbnails[::1 if write_all else -1]:
+ for t in thumbnails[::-1]:
thumb_ext = determine_ext(t['url'], 'jpg')
suffix = '%s.' % t['id'] if multiple else ''
thumb_display_id = '%s ' % t['id'] if multiple else ''