mkdir -p completions/fish
$(PYTHON) devscripts/fish-completion.py
-_EXTRACTOR_FILES = $(shell find yt_dlp/extractor -iname '*.py' -and -not -iname 'lazy_extractors.py')
+_EXTRACTOR_FILES = $(shell find yt_dlp/extractor -name '*.py' -and -not -name 'lazy_extractors.py')
yt_dlp/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
$(PYTHON) devscripts/make_lazy_extractors.py $@
* **Output template improvements**: Output templates can now have date-time formatting, numeric offsets, object traversal etc. See [output template](#output-template) for details. Even more advanced operations can also be done with the help of `--parse-metadata` and `--replace-in-metadata`
-* **Other new options**: Many new options have been added such as `--concat-playlist`, `--print`, `--wait-for-video`, `--sleep-requests`, `--convert-thumbnails`, `--write-link`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc
+* **Other new options**: Many new options have been added such as `--alias`, `--print`, `--concat-playlist`, `--wait-for-video`, `--retry-sleep`, `--sleep-requests`, `--convert-thumbnails`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc
* **Improvements**: Regex and other operators in `--format`/`--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio, multiple `--config-locations`, `--exec` at different stages, etc
For ease of use, a few more compat options are available:
-* `--compat-options all`: Use all compat options
+* `--compat-options all`: Use all compat options (Do NOT use)
* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams`
* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect`
--no-remove-chapters Do not remove any chapters from the file
(default)
--force-keyframes-at-cuts Force keyframes around chapters when
- removing/splitting them. The resulting video
- may have fewer artifacts around the cuts,
- but is very slow due to needing a re-encode
+ removing/splitting them. This is slow due to
+ needing a re-encode, but the resulting video
+ may have fewer artifacts around the cuts
--no-force-keyframes-at-cuts Do not force keyframes around the chapters
when cutting/splitting (default)
--use-postprocessor NAME[:ARGS]
OPTIONS_END = 'CONFIGURATION'
EPILOG_START = 'See full documentation'
+DISABLE_PATCH = object()
+
def take_section(text, start=None, end=None, *, shift=0):
return text[
def apply_patch(text, patch):
- return re.sub(*patch, text)
+ return text if patch[0] is DISABLE_PATCH else re.sub(*patch, text)
options = take_section(sys.stdin.read(), f'\n {OPTIONS_START}', f'\n{EPILOG_START}', shift=1)
rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s',
lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n'))
),
- # This creates issues with prepare_manpage
- # ( # Avoid newline when a space is available b/w switch and description
- # r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
- # r'\1 '
- # ),
+ ( # Do not split "words"
+ rf'(?m)({delim}\S+)+$',
+ lambda mobj: ''.join((delim, mobj.group(0).replace(delim, '')))
+ ),
+ ( # Avoid newline when a space is available b/w switch and description
+ DISABLE_PATCH, # This creates issues with prepare_manpage
+ r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
+ r'\1 '
+ ),
)
with open(README_FILE, encoding='utf-8') as f:
-#!/bin/sh
+#!/usr/bin/env sh
if [ -z $1 ]; then
test_set='test'
[wheel]
universal = true
+
[flake8]
-exclude = build,venv,.tox,.git
+exclude = build,venv,.tox,.git,.pytest_cache
ignore = E402,E501,E731,E741,W503
+max_line_length = 120
per_file_ignores =
- ./devscripts/lazy_load_template.py: F401
+ devscripts/lazy_load_template.py: F401
+
[tool:pytest]
addopts = -ra -v --strict-markers
markers =
download
+
[tox:tox]
skipsdist = true
envlist = py{36,37,38,39,310},pypy{36,37,38,39}
setenv =
# PYTHONWARNINGS = error # Catches PIP's warnings too
+
[isort]
py_version = 36
multi_line_output = VERTICAL_HANGING_INDENT
-#!/bin/sh
+#!/usr/bin/env sh
exec "${PYTHON:-python3}" -bb -Werror -Xdev "$(dirname "$(realpath "$0")")/yt_dlp/__main__.py" "$@"
from .cache import Cache
from .compat import (
+ HAS_LEGACY as compat_has_legacy,
compat_get_terminal_size,
compat_os_name,
compat_shlex_quote,
for msg in self.params.get('_deprecation_warnings', []):
self.deprecation_warning(msg)
- if 'list-formats' in self.params.get('compat_opts', []):
+ self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
+ if not compat_has_legacy:
+ self.params['compat_opts'].add('no-compat-legacy')
+ if 'list-formats' in self.params['compat_opts']:
self.params['listformats_table'] = False
if 'overwrites' not in self.params and self.params.get('nooverwrites') is not None:
"""Print message to stdout"""
if quiet is not None:
self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. Use "YoutubeDL.to_screen" instead')
- self._write_string(
- '%s%s' % (self._bidi_workaround(message), ('' if skip_eol else '\n')),
- self._out_files.out)
+ if skip_eol is not False:
+ self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument skip_eol. Use "YoutubeDL.to_screen" instead')
+ self._write_string(self._bidi_workaround(message), self._out_files.out)
def to_screen(self, message, skip_eol=False, quiet=None):
"""Print message to screen if not in quiet mode"""
'''Log debug message or Print message to stderr'''
if not self.params.get('verbose', False):
return
- message = '[debug] %s' % message
+ message = f'[debug] {message}'
if self.params.get('logger'):
self.params['logger'].debug(message)
else:
def filename_sanitizer(key, value, restricted=self.params.get('restrictfilenames')):
return sanitize_filename(str(value), restricted=restricted, is_id=(
bool(re.search(r'(^|[_.])id(\.|$)', key))
- if 'filename-sanitization' in self.params.get('compat_opts', [])
+ if 'filename-sanitization' in self.params['compat_opts']
else NO_DEFAULT))
sanitizer = sanitize if callable(sanitize) else filename_sanitizer
max_failures = self.params.get('skip_playlist_after_errors') or float('inf')
for i, entry_tuple in enumerate(entries, 1):
playlist_index, entry = entry_tuple
- if 'playlist-index' in self.params.get('compat_opts', []):
+ if 'playlist-index' in self.params['compat_opts']:
playlist_index = playlistitems[i - 1] if playlistitems else i + playliststart - 1
self.to_screen('[download] Downloading video %s of %s' % (
self._format_screen(i, self.Styles.ID), self._format_screen(n_entries, self.Styles.EMPHASIS)))
temp_file.close()
try:
success, _ = self.dl(temp_file.name, f, test=True)
- except (DownloadError, IOError, OSError, ValueError) + network_exceptions:
+ except (DownloadError, OSError, ValueError) + network_exceptions:
success = False
finally:
if os.path.exists(temp_file.name):
compat = (
prefer_best
or self.params.get('allow_multiple_audio_streams', False)
- or 'format-spec' in self.params.get('compat_opts', []))
+ or 'format-spec' in self.params['compat_opts'])
return (
'best/bestvideo+bestaudio' if prefer_best
write_debug('Plugins: %s' % [
'%s%s' % (klass.__name__, '' if klass.__name__ == name else f' as {name}')
for name, klass in itertools.chain(plugin_extractors.items(), plugin_postprocessors.items())])
- if self.params.get('compat_opts'):
- write_debug('Compatibility options: %s' % ', '.join(self.params.get('compat_opts')))
+ if self.params['compat_opts']:
+ write_debug('Compatibility options: %s' % ', '.join(self.params['compat_opts']))
if source == 'source':
try:
# XXX: Implement this the same way as other DeprecationWarnings without circular import
-passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
- DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=2))
+try:
+ passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
+ DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=2))
+ HAS_LEGACY = True
+except ModuleNotFoundError:
+ # Keep working even without _legacy module
+ HAS_LEGACY = False
del passthrough_module
-from ..compat import compat_str
from ..utils import NO_DEFAULT, determine_protocol
info_dict['protocol'] = protocol
downloaders = params.get('external_downloader')
external_downloader = (
- downloaders if isinstance(downloaders, compat_str) or downloaders is None
+ downloaders if isinstance(downloaders, str) or downloaders is None
else downloaders.get(shorten_protocol_name(protocol, True), downloaders.get('default')))
if external_downloader is None:
if ip_block:
self._x_forwarded_for_ip = GeoUtils.random_ipv4(ip_block)
- self._downloader.write_debug(
- '[debug] Using fake IP %s as X-Forwarded-For' % self._x_forwarded_for_ip)
+ self.write_debug(f'Using fake IP {self._x_forwarded_for_ip} as X-Forwarded-For')
return
# Path 2: bypassing based on country code
auth_cookie = self._get_cookies('https://curiositystream.com').get('auth_token')
if auth_cookie:
self.write_debug('Obtained auth_token cookie')
- self._auth_token = cookie.value
+ self._auth_token = auth_cookie.value
if self._auth_token:
headers['X-Auth-Token'] = self._auth_token
result = self._download_json(
import re
from .common import InfoExtractor
-from ..compat import (
- compat_parse_qs,
-)
+from ..compat import compat_parse_qs
from ..dependencies import websockets
from ..utils import (
ExtractorError,
'User-Agent': self.get_param('http_headers')['User-Agent'],
})
- self.write_debug('[debug] Sending HLS server request')
+ self.write_debug('Sending HLS server request')
while True:
recv = ws.recv()
if not data or not isinstance(data, dict):
continue
if data.get('name') == '_response_' and data.get('id') == 1:
- self.write_debug('[debug] Goodbye.')
+ self.write_debug('Goodbye')
playlist_data = data
break
- elif self._downloader.params.get('verbose', False):
- if len(recv) > 100:
- recv = recv[:100] + '...'
- self.to_screen('[debug] Server said: %s' % recv)
+ self.write_debug('Server said: %s%s' % (recv[:100], '...' if len(recv) > 100 else ''))
if not playlist_data:
raise ExtractorError('Unable to fetch HLS playlist info via WebSocket')
action='store_true', dest='force_keyframes_at_cuts', default=False,
help=(
'Force keyframes around chapters when removing/splitting them. '
- 'The resulting video may have fewer artifacts around the cuts, '
- 'but is very slow due to needing a re-encode'))
+ 'This is slow due to needing a re-encode, but '
+ 'the resulting video may have fewer artifacts around the cuts'))
postproc.add_option(
'--no-force-keyframes-at-cuts',
action='store_false', dest='force_keyframes_at_cuts',
def report_progress(self, s):
s['_default_template'] = '%(postprocessor)s %(status)s' % s
+ if not self._downloader:
+ return
progress_dict = s.copy()
progress_dict.pop('info_dict')
progress_template = self.get_param('progress_template', {})
tmpl = progress_template.get('postprocess')
if tmpl:
- self._downloader.to_stdout(self._downloader.evaluate_outtmpl(tmpl, progress_dict))
+ self._downloader.to_screen(
+ self._downloader.evaluate_outtmpl(tmpl, progress_dict), skip_eol=True, quiet=False)
self._downloader.to_console_title(self._downloader.evaluate_outtmpl(
progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s',
self._prefer_ffmpeg = self.get_param('prefer_ffmpeg', True)
self._paths = self._determine_executables()
- def check_version(self):
- if not self.available:
- raise FFmpegPostProcessorError('ffmpeg not found. Please install or provide the path using --ffmpeg-location')
-
- required_version = '10-0' if self.basename == 'avconv' else '1.0'
- if is_outdated_version(self._version, required_version):
- self.report_warning(f'Your copy of {self.basename} is outdated, update {self.basename} '
- f'to version {required_version} or newer if you encounter any errors')
-
@staticmethod
def get_versions_and_features(downloader=None):
pp = FFmpegPostProcessor(downloader)
if ext in ('mp4', 'mov', 'm4a'):
yield from ('-c:s', 'mov_text')
+ def check_version(self):
+ if not self.available:
+ raise FFmpegPostProcessorError('ffmpeg not found. Please install or provide the path using --ffmpeg-location')
+
+ required_version = '10-0' if self.basename == 'avconv' else '1.0'
+ if is_outdated_version(self._version, required_version):
+ self.report_warning(f'Your copy of {self.basename} is outdated, update {self.basename} '
+ f'to version {required_version} or newer if you encounter any errors')
+
def get_audio_codec(self, path):
if not self.probe_available and not self.available:
raise PostProcessingError('ffprobe and ffmpeg not found. Please install or provide the path using --ffmpeg-location')
# Ref: https://github.com/yt-dlp/yt-dlp/issues/3124
raise LockingUnsupportedError()
stream = locked_file(filename, open_mode, block=False).__enter__()
- except LockingUnsupportedError:
+ except OSError:
stream = open(filename, open_mode)
- return (stream, filename)
+ return stream, filename
except OSError as err:
if attempt or err.errno in (errno.EACCES,):
raise
def process_communicate_or_kill(p, *args, **kwargs):
- try:
- return p.communicate(*args, **kwargs)
- except BaseException: # Including KeyboardInterrupt
- p.kill()
- p.wait()
- raise
+ write_string('DeprecationWarning: yt_dlp.utils.process_communicate_or_kill is deprecated '
+ 'and may be removed in a future version. Use yt_dlp.utils.Popen.communicate_or_kill instead')
+ return Popen.communicate_or_kill(p, *args, **kwargs)
class Popen(subprocess.Popen):
super().__init__(*args, **kwargs, startupinfo=self._startupinfo)
def communicate_or_kill(self, *args, **kwargs):
- return process_communicate_or_kill(self, *args, **kwargs)
+ try:
+ return self.communicate(*args, **kwargs)
+ except BaseException: # Including KeyboardInterrupt
+ self.kill()
+ self.wait()
+ raise
def get_subprocess_encoding():
context.options |= 4 # SSL_OP_LEGACY_SERVER_CONNECT
# Allow use of weaker ciphers in Python 3.10+. See https://bugs.python.org/issue43998
context.set_ciphers('DEFAULT')
+
context.verify_mode = ssl.CERT_REQUIRED if opts_check_certificate else ssl.CERT_NONE
if opts_check_certificate:
if has_certifi and 'no-certifi' not in params.get('compat_opts', []):
context.load_verify_locations(cafile=certifi.where())
- else:
- try:
- context.load_default_certs()
- # Work around the issue in load_default_certs when there are bad certificates. See:
- # https://github.com/yt-dlp/yt-dlp/issues/1060,
- # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
- except ssl.SSLError:
- # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
- if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
- for storename in ('CA', 'ROOT'):
- _ssl_load_windows_store_certs(context, storename)
- context.set_default_verify_paths()
+ try:
+ context.load_default_certs()
+ # Work around the issue in load_default_certs when there are bad certificates. See:
+ # https://github.com/yt-dlp/yt-dlp/issues/1060,
+ # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
+ except ssl.SSLError:
+ # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
+ if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
+ for storename in ('CA', 'ROOT'):
+ _ssl_load_windows_store_certs(context, storename)
+ context.set_default_verify_paths()
+
client_certfile = params.get('client_certificate')
if client_certfile:
try:
@functools.cache
def get_windows_version():
- ''' Get Windows version. None if it's not running on Windows '''
+ ''' Get Windows version. returns () if it's not running on Windows '''
if compat_os_name == 'nt':
return version_tuple(platform.win32_ver()[1])
else:
- return None
+ return ()
def write_string(s, out=None, encoding=None):
if compat_os_name == 'nt' and supports_terminal_sequences(out):
s = re.sub(r'([\r\n]+)', r' \1', s)
- enc = None
+ enc, buffer = None, out
if 'b' in getattr(out, 'mode', ''):
enc = encoding or preferredencoding()
elif hasattr(out, 'buffer'):
- out = out.buffer
+ buffer = out.buffer
enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
- out.write(s.encode(enc, 'ignore') if enc else s)
+ buffer.write(s.encode(enc, 'ignore') if enc else s)
out.flush()
return compat_struct_pack('%dB' % len(xs), *xs)
-class LockingUnsupportedError(IOError):
+class LockingUnsupportedError(OSError):
msg = 'File locking is not supported on this platform'
def __init__(self):
@functools.cache
def supports_terminal_sequences(stream):
if compat_os_name == 'nt':
- if not WINDOWS_VT_MODE or get_windows_version() < (10, 0, 10586):
+ if not WINDOWS_VT_MODE:
return False
elif not os.getenv('TERM'):
return False
def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075
- if compat_os_name != 'nt':
+ if get_windows_version() < (10, 0, 10586):
return
global WINDOWS_VT_MODE
startupinfo = subprocess.STARTUPINFO()
projects / yt-dlp.git / commitdiff
