from .extractor.common import UnsupportedURLIE
from .extractor.openload import PhantomJSwrapper
from .minicurses import format_text
-from .networking import Request, RequestDirector
-from .networking.common import _REQUEST_HANDLERS
+from .networking import HEADRequest, Request, RequestDirector
+from .networking.common import _REQUEST_HANDLERS, _RH_PREFERENCES
from .networking.exceptions import (
HTTPError,
NoSupportingHandlers,
RequestError,
SSLError,
_CompatHTTPError,
+ network_exceptions,
)
from .plugins import directories as plugin_directories
from .postprocessor import _PLUGIN_CLASSES as plugin_pps
get_postprocessor,
)
from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
-from .update import REPOSITORY, current_git_head, detect_variant
+from .update import REPOSITORY, _get_system_deprecation, _make_label, current_git_head, detect_variant
from .utils import (
DEFAULT_OUTTMPL,
IDENTITY,
ExtractorError,
FormatSorter,
GeoRestrictedError,
- HEADRequest,
ISO3166Utils,
LazyList,
MaxDownloadsReached,
locked_file,
make_archive_id,
make_dir,
- network_exceptions,
number_of_digits,
orderedSet,
orderedSet_from_options,
sanitize_filename,
sanitize_path,
sanitize_url,
- std_headers,
str_or_none,
strftime_or_none,
subtitles_filename,
HTTPHeaderDict,
clean_headers,
clean_proxies,
+ std_headers,
)
-from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
+from .version import CHANNEL, ORIGIN, RELEASE_GIT_HEAD, VARIANT, __version__
if compat_os_name == 'nt':
import ctypes
'selected' (check selected formats),
or None (check only if requested by extractor)
paths: Dictionary of output paths. The allowed keys are 'home'
- 'temp' and the keys of OUTTMPL_TYPES (in utils.py)
+ 'temp' and the keys of OUTTMPL_TYPES (in utils/_utils.py)
outtmpl: Dictionary of templates for output names. Allowed keys
- are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
+ are 'default' and the keys of OUTTMPL_TYPES (in utils/_utils.py).
For compatibility with youtube-dl, a single string can also be used
outtmpl_na_placeholder: Placeholder for unavailable meta fields.
restrictfilenames: Do not allow "&" and spaces in file names
overwrites: Overwrite all video and metadata files if True,
overwrite only non-video files if None
and don't overwrite any file if False
- For compatibility with youtube-dl,
- "nooverwrites" may also be used instead
playlist_items: Specific indices of playlist to download.
playlistrandom: Download playlist items in random order.
lazy_playlist: Process playlist entries as they are received.
asked whether to download the video.
- Raise utils.DownloadCancelled(msg) to abort remaining
downloads when a video is rejected.
- match_filter_func in utils.py is one example for this.
+ match_filter_func in utils/_utils.py is one example for this.
color: A Dictionary with output stream names as keys
and their respective color policy as values.
Can also just be a single color policy,
You can reduce network I/O by disabling it if you don't
care about HLS. (only for youtube)
no_color: Same as `color='no_color'`
+ no_overwrites: Same as `overwrites=False`
"""
_NUMERIC_FIELDS = {
'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns',
'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
- 'preference', 'language', 'language_preference', 'quality', 'source_preference',
+ 'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies',
'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options',
'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'
}
self._playlist_level = 0
self._playlist_urls = set()
self.cache = Cache(self)
+ self.__header_cookies = []
stdout = sys.stderr if self.params.get('logtostderr') else sys.stdout
self._out_files = Namespace(
if self.params.get('no_color'):
if self.params.get('color') is not None:
- self.report_warning('Overwriting params from "color" with "no_color"')
+ self.params.setdefault('_warnings', []).append(
+ 'Overwriting params from "color" with "no_color"')
self.params['color'] = 'no_color'
term_allow_color = os.environ.get('TERM', '').lower() != 'dumb'
policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False)
if policy in ('auto', None):
return term_allow_color and supports_terminal_sequences(stream)
- assert policy in ('always', 'never', 'no_color')
+ assert policy in ('always', 'never', 'no_color'), policy
return {'always': True, 'never': False}.get(policy, policy)
self._allow_colors = Namespace(**{
for name, stream in self._out_files.items_ if name != 'console'
})
- # The code is left like this to be reused for future deprecations
- MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 7)
- current_version = sys.version_info[:2]
- if current_version < MIN_RECOMMENDED:
- msg = ('Support for Python version %d.%d has been deprecated. '
- 'See https://github.com/yt-dlp/yt-dlp/issues/3764 for more details.'
- '\n You will no longer receive updates on this version')
- if current_version < MIN_SUPPORTED:
- msg = 'Python version %d.%d is no longer supported'
- self.deprecated_feature(
- f'{msg}! Please update to Python %d.%d or above' % (*current_version, *MIN_RECOMMENDED))
+ system_deprecation = _get_system_deprecation()
+ if system_deprecation:
+ self.deprecated_feature(system_deprecation.replace('\n', '\n '))
if self.params.get('allow_unplayable_formats'):
self.report_warning(
self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers'))
- self._request_director = self.build_request_director(
- sorted(_REQUEST_HANDLERS.values(), key=lambda rh: rh.RH_NAME.lower()))
+ self._load_cookies(self.params['http_headers'].get('Cookie')) # compat
+ self.params['http_headers'].pop('Cookie', None)
+ self._request_director = self.build_request_director(_REQUEST_HANDLERS.values(), _RH_PREFERENCES)
+
if auto_init and auto_init != 'no_verbose_header':
self.print_debug_header()
- self.__header_cookies = []
- self._load_cookies(traverse_obj(self.params.get('http_headers'), 'cookie', casesense=False)) # compat
-
def check_deprecated(param, option, suggestion):
if self.params.get(param) is not None:
self.report_warning(f'{option} is deprecated. Use {suggestion} instead')
def save_cookies(self):
if self.params.get('cookiefile') is not None:
- self.cookiejar.save(ignore_discard=True, ignore_expires=True)
+ self.cookiejar.save()
def __exit__(self, *args):
self.restore_console_title()
else:
break
- fmt = outer_mobj.group('format')
- if fmt == 's' and value is not None and last_field in field_size_compat_map.keys():
- fmt = f'0{field_size_compat_map[last_field]:d}d'
-
if None not in (value, replacement):
try:
value = replacement_formatter.format(replacement, value)
except ValueError:
value, default = None, na
+ fmt = outer_mobj.group('format')
+ if fmt == 's' and last_field in field_size_compat_map.keys() and isinstance(value, int):
+ fmt = f'0{field_size_compat_map[last_field]:d}d'
+
flags = outer_mobj.group('conversion') or ''
str_fmt = f'{fmt[:-1]}s'
if value is None:
return ret
if self.in_download_archive(info_dict):
- reason = '%s has already been recorded in the archive' % video_title
+ reason = ''.join((
+ format_field(info_dict, 'id', f'{self._format_screen("%s", self.Styles.ID)}: '),
+ format_field(info_dict, 'title', f'{self._format_screen("%s", self.Styles.EMPHASIS)} '),
+ 'has already been recorded in the archive'))
break_opt, break_err = 'break_on_existing', ExistingVideoReached
else:
try:
temp_id = ie.get_temp_id(url)
if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': key}):
- self.to_screen(f'[{key}] {temp_id}: has already been recorded in the archive')
+ self.to_screen(f'[download] {self._format_screen(temp_id, self.Styles.ID)}: '
+ 'has already been recorded in the archive')
if self.params.get('break_on_existing', False):
raise ExistingVideoReached()
break
self.to_screen('')
raise
- def _load_cookies(self, data, *, from_headers=True):
+ def _load_cookies(self, data, *, autoscope=True):
"""Loads cookies from a `Cookie` header
This tries to work around the security vulnerability of passing cookies to every domain.
See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
- The unscoped cookies are saved for later to be stored in the jar with a limited scope.
@param data The Cookie header as string to load the cookies from
- @param from_headers If `False`, allows Set-Cookie syntax in the cookie string (at least a domain will be required)
+ @param autoscope If `False`, scope cookies using Set-Cookie syntax and error for cookie without domains
+ If `True`, save cookies for later to be stored in the jar with a limited scope
+ If a URL, save cookies in the jar with the domain of the URL
"""
for cookie in LenientSimpleCookie(data).values():
- if from_headers and any(cookie.values()):
+ if autoscope and any(cookie.values()):
raise ValueError('Invalid syntax in Cookie Header')
domain = cookie.get('domain') or ''
if domain:
self.cookiejar.set_cookie(prepared_cookie)
- elif from_headers:
+ elif autoscope is True:
self.deprecated_feature(
'Passing cookies as a header is a potential security risk; '
'they will be scoped to the domain of the downloaded urls. '
'Please consider loading cookies from a file or browser instead.')
self.__header_cookies.append(prepared_cookie)
+ elif autoscope:
+ self.report_warning(
+ 'The extractor result contains an unscoped cookie as an HTTP header. '
+ f'If you are using yt-dlp with an input URL{bug_reports_message(before=",")}',
+ only_once=True)
+ self._apply_header_cookies(autoscope, [prepared_cookie])
else:
self.report_error('Unscoped cookies are not allowed; please specify some sort of scoping',
tb=False, is_error=False)
- def _apply_header_cookies(self, url):
+ def _apply_header_cookies(self, url, cookies=None):
"""Applies stray header cookies to the provided url
This loads header cookies and scopes them to the domain provided in `url`.
if not parsed.hostname:
return
- for cookie in map(copy.copy, self.__header_cookies):
+ for cookie in map(copy.copy, cookies or self.__header_cookies):
cookie.domain = f'.{parsed.hostname}'
self.cookiejar.set_cookie(cookie)
return new_dict
def _check_formats(formats):
- if (self.params.get('check_formats') is not None
+ if self.params.get('check_formats') == 'selected':
+ yield from self._check_formats(formats)
+ return
+ elif (self.params.get('check_formats') is not None
or self.params.get('allow_unplayable_formats')):
yield from formats
return
- elif self.params.get('check_formats') == 'selected':
- yield from self._check_formats(formats)
- return
for f in formats:
- if f.get('has_drm'):
+ if f.get('has_drm') or f.get('__needs_testing'):
yield from self._check_formats([f])
else:
yield f
parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
return _build_selector_function(parsed_selector)
- def _calc_headers(self, info_dict):
+ def _calc_headers(self, info_dict, load_cookies=False):
res = HTTPHeaderDict(self.params['http_headers'], info_dict.get('http_headers'))
clean_headers(res)
+
+ if load_cookies: # For --load-info-json
+ self._load_cookies(res.get('Cookie'), autoscope=info_dict['url']) # compat
+ self._load_cookies(info_dict.get('cookies'), autoscope=False)
+ # The `Cookie` header is removed to prevent leaks and unscoped cookies.
+ # See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
+ res.pop('Cookie', None)
cookies = self.cookiejar.get_cookies_for_url(info_dict['url'])
if cookies:
encoder = LenientSimpleCookie()
# Working around out-of-range timestamp values (e.g. negative ones on Windows,
# see http://bugs.python.org/issue1646728)
with contextlib.suppress(ValueError, OverflowError, OSError):
- upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
+ upload_date = datetime.datetime.fromtimestamp(info_dict[ts_key], datetime.timezone.utc)
info_dict[date_key] = upload_date.strftime('%Y%m%d')
live_keys = ('is_live', 'was_live')
format['dynamic_range'] = 'SDR'
if format.get('aspect_ratio') is None:
format['aspect_ratio'] = try_call(lambda: round(format['width'] / format['height'], 2))
- if (not format.get('manifest_url') # For fragmented formats, "tbr" is often max bitrate and not average
+ # For fragmented formats, "tbr" is often max bitrate and not average
+ if (('manifest-filesize-approx' in self.params['compat_opts'] or not format.get('manifest_url'))
and info_dict.get('duration') and format.get('tbr')
and not format.get('filesize') and not format.get('filesize_approx')):
format['filesize_approx'] = int(info_dict['duration'] * format['tbr'] * (1024 / 8))
- format['http_headers'] = self._calc_headers(collections.ChainMap(format, info_dict))
+ format['http_headers'] = self._calc_headers(collections.ChainMap(format, info_dict), load_cookies=True)
+
+ # Safeguard against old/insecure infojson when using --load-info-json
+ if info_dict.get('http_headers'):
+ info_dict['http_headers'] = HTTPHeaderDict(info_dict['http_headers'])
+ info_dict['http_headers'].pop('Cookie', None)
# This is copied to http_headers by the above _calc_headers and can now be removed
if '__x_forwarded_for_ip' in info_dict:
fd, success = None, True
if info_dict.get('protocol') or info_dict.get('url'):
fd = get_suitable_downloader(info_dict, self.params, to_stdout=temp_filename == '-')
- if fd is not FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and (
+ if fd != FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and (
info_dict.get('section_start') or info_dict.get('section_end')):
msg = ('This format cannot be partially downloaded' if FFmpegFD.available()
else 'You have requested downloading the video partially, but ffmpeg is not installed')
) for pp in self._pps['post_process'])
if not postprocessed_by_ffmpeg:
- ffmpeg_fixup(ext == 'm4a' and info_dict.get('container') == 'm4a_dash',
+ ffmpeg_fixup(fd != FFmpegFD and ext == 'm4a'
+ and info_dict.get('container') == 'm4a_dash',
'writing DASH m4a. Only some players support this container',
FFmpegFixupM4aPP)
ffmpeg_fixup(downloader == 'hlsnative' and not self.params.get('hls_use_mpegts')
infos = [self.sanitize_info(info, self.params.get('clean_infojson', True))
for info in variadic(json.loads('\n'.join(f)))]
for info in infos:
- self._load_cookies(info.get('cookies'), from_headers=False)
- self._load_cookies(traverse_obj(info.get('http_headers'), 'Cookie', casesense=False)) # compat
try:
self.__download_wrapper(self.process_ie_result)(info, download=True)
except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e:
'version': __version__,
'current_git_head': current_git_head(),
'release_git_head': RELEASE_GIT_HEAD,
- 'repository': REPOSITORY,
+ 'repository': ORIGIN,
})
if remove_private_keys:
reject = lambda k, v: v is None or k.startswith('__') or k in {
'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
'entries', 'filepath', '_filename', 'filename', 'infojson_filename', 'original_url',
- 'playlist_autonumber', '_format_sort_fields',
+ 'playlist_autonumber',
}
else:
reject = lambda k, v: False
source += '*'
klass = type(self)
write_debug(join_nonempty(
- f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
- f'{CHANNEL}@{__version__}',
+ f'{REPOSITORY.rpartition("/")[2]} version',
+ _make_label(ORIGIN, CHANNEL.partition('@')[2] or __version__, __version__),
f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '',
'' if source == 'unknown' else f'({source})',
'' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',
})) or 'none'))
write_debug(f'Proxy map: {self.proxies}')
- # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers)}')
+ write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}')
for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items():
display_list = ['%s%s' % (
klass.__name__, '' if klass.__name__ == name else f' as {name}')
"""
Get a urllib OpenerDirector from the Urllib handler (deprecated).
"""
- self.deprecation_warning('YoutubeDL._opener() is deprecated, use YoutubeDL.urlopen()')
+ self.deprecation_warning('YoutubeDL._opener is deprecated, use YoutubeDL.urlopen()')
handler = self._request_director.handlers['Urllib']
return handler._get_instance(cookiejar=self.cookiejar, proxies=self.proxies)
if isinstance(req, str):
req = Request(req)
elif isinstance(req, urllib.request.Request):
+ self.deprecation_warning(
+ 'Passing a urllib.request.Request object to YoutubeDL.urlopen() is deprecated. '
+ 'Use yt_dlp.networking.common.Request instead.')
req = urllib_req_to_req(req)
assert isinstance(req, Request)
raise RequestError(
'file:// URLs are disabled by default in yt-dlp for security reasons. '
'Use --enable-file-urls to enable at your own risk.', cause=ue) from ue
+ if 'unsupported proxy type: "https"' in ue.msg.lower():
+ raise RequestError(
+ 'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests')
raise
except SSLError as e:
if 'UNSAFE_LEGACY_RENEGOTIATION_DISABLED' in str(e):
except HTTPError as e: # TODO: Remove in a future release
raise _CompatHTTPError(e) from e
- def build_request_director(self, handlers):
+ def build_request_director(self, handlers, preferences=None):
logger = _YDLLogger(self)
- headers = self.params.get('http_headers').copy()
+ headers = self.params['http_headers'].copy()
proxies = self.proxies.copy()
clean_headers(headers)
clean_proxies(proxies, headers)
'verbose': 'debug_printtraffic',
'source_address': 'source_address',
'timeout': 'socket_timeout',
- 'legacy_ssl_support': 'legacy_server_connect',
+ 'legacy_ssl_support': 'legacyserverconnect',
'enable_file_urls': 'enable_file_urls',
'client_cert': {
'client_certificate': 'client_certificate',
},
}),
))
+ director.preferences.update(preferences or [])
+ if 'prefer-legacy-http-handler' in self.params['compat_opts']:
+ director.preferences.add(lambda rh, _: 500 if rh.RH_KEY == 'Urllib' else 0)
return director
def encode(self, s):
return ret
def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None):
- ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename) '''
+ ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename); or None if error '''
write_all = self.params.get('write_all_thumbnails', False)
thumbnails, ret = [], []
if write_all or self.params.get('writethumbnail', False):
self.write_debug(f'Skipping writing {label} thumbnail')
return ret
+ if thumbnails and not self._ensure_dir_exists(filename):
+ return None
+
for idx, t in list(enumerate(thumbnails))[::-1]:
thumb_ext = (f'{t["id"]}.' if multiple else '') + determine_ext(t['url'], 'jpg')
thumb_display_id = f'{label} thumbnail {t["id"]}'
ret.append((thumb_filename, thumb_filename_final))
t['filepath'] = thumb_filename
except network_exceptions as err:
- if isinstance(err, urllib.error.HTTPError) and err.code == 404:
+ if isinstance(err, HTTPError) and err.status == 404:
self.to_screen(f'[info] {thumb_display_id.title()} does not exist')
else:
self.report_warning(f'Unable to download {thumb_display_id}: {err}')