1 from __future__
import annotations
12 from dataclasses
import dataclass
13 from zipimport
import zipimporter
15 from .compat
import functools
# isort: split
16 from .compat
import compat_realpath
, compat_shlex_quote
17 from .networking
import Request
18 from .networking
.exceptions
import HTTPError
, network_exceptions
29 from .version
import (
39 'stable': 'yt-dlp/yt-dlp',
40 'nightly': 'yt-dlp/yt-dlp-nightly-builds',
41 'master': 'yt-dlp/yt-dlp-master-builds',
43 REPOSITORY
= UPDATE_SOURCES
['stable']
44 _INVERSE_UPDATE_SOURCES
= {value: key for key, value in UPDATE_SOURCES.items()}
46 _VERSION_RE
= re
.compile(r
'(\d+\.)*\d+')
47 _HASH_PATTERN
= r
'[\da-f]{40}'
48 _COMMIT_RE
= re
.compile(rf
'Generated from: https://(?:[^/?#]+/){{3}}commit/(?P<hash>{_HASH_PATTERN})')
50 API_BASE_URL
= 'https://api.github.com/repos'
52 # Backwards compatibility variables for the current channel
53 API_URL
= f
'{API_BASE_URL}/{REPOSITORY}/releases'
57 def _get_variant_and_executable_path():
58 """@returns (variant, executable_path)"""
59 if getattr(sys
, 'frozen', False):
61 if not hasattr(sys
, '_MEIPASS'):
63 elif sys
._MEIPASS
== os
.path
.dirname(path
):
64 return f
'{sys.platform}_dir', path
65 elif sys
.platform
== 'darwin':
66 machine
= '_legacy' if version_tuple(platform
.mac_ver()[0]) < (10, 15) else ''
68 machine
= f
'_{platform.machine().lower()}'
69 # Ref: https://en.wikipedia.org/wiki/Uname#Examples
70 if machine
[1:] in ('x86', 'x86_64', 'amd64', 'i386', 'i686'):
71 machine
= '_x86' if platform
.architecture()[0][:2] == '32' else ''
72 return f
'{remove_end(sys.platform, "32")}{machine}_exe', path
74 path
= os
.path
.dirname(__file__
)
75 if isinstance(__loader__
, zipimporter
):
76 return 'zip', os
.path
.join(path
, '..')
77 elif (os
.path
.basename(sys
.argv
[0]) in ('__main__.py', '-m')
78 and os
.path
.exists(os
.path
.join(path
, '../.git/HEAD'))):
80 return 'unknown', path
84 return VARIANT
or _get_variant_and_executable_path()[0]
88 def current_git_head():
89 if detect_variant() != 'source':
91 with contextlib
.suppress(Exception):
92 stdout
, _
, _
= Popen
.run(
93 ['git', 'rev-parse', '--short', 'HEAD'],
94 text
=True, cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)),
95 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
96 if re
.fullmatch('[0-9a-f]+', stdout
.strip()):
102 'py2exe': '_min.exe',
104 'win_x86_exe': '_x86.exe',
105 'darwin_exe': '_macos',
106 'darwin_legacy_exe': '_macos_legacy',
107 'linux_exe': '_linux',
108 'linux_aarch64_exe': '_linux_aarch64',
109 'linux_armv7l_exe': '_linux_armv7l',
112 _NON_UPDATEABLE_REASONS
= {
113 **{variant: None for variant in _FILE_SUFFIXES}
, # Updatable
114 **{variant: f'Auto-update is not supported for unpackaged {name} executable
; Re
-download the latest release
'
115 for variant, name in {'win32_dir': 'Windows', 'darwin_dir': 'MacOS', 'linux_dir': 'Linux'}.items()},
116 'source
': 'You cannot update when running
from source code
; Use git to pull the latest changes
',
117 'unknown
': 'You installed yt
-dlp
with a package manager
or setup
.py
; Use that to update
',
118 'other
': 'You are using an unofficial build of yt
-dlp
; Build the executable again
',
122 def is_non_updateable():
125 return _NON_UPDATEABLE_REASONS.get(
126 detect_variant(), _NON_UPDATEABLE_REASONS['unknown
' if VARIANT else 'other
'])
129 def _get_binary_name():
130 return format_field(_FILE_SUFFIXES, detect_variant(), template='yt
-dlp
%s', ignore=None, default=None)
133 def _get_system_deprecation():
134 MIN_SUPPORTED, MIN_RECOMMENDED = (3, 8), (3, 8)
136 if sys.version_info > MIN_RECOMMENDED:
139 major, minor = sys.version_info[:2]
140 if sys.version_info < MIN_SUPPORTED:
141 msg = f'Python version {major}
.{minor}
is no longer supported
'
143 msg = (f'Support
for Python version {major}
.{minor} has been deprecated
. '
144 '\nYou may stop receiving updates on this version at any time
')
146 major, minor = MIN_RECOMMENDED
147 return f'{msg}
! Please update to Python {major}
.{minor}
or above
'
150 def _sha256_file(path):
152 mv = memoryview(bytearray(128 * 1024))
153 with open(os.path.realpath(path), 'rb
', buffering=0) as f:
154 for n in iter(lambda: f.readinto(mv), 0):
159 def _make_label(origin, tag, version=None):
161 channel = _INVERSE_UPDATE_SOURCES.get(origin, origin)
164 label = f'{channel}
@{tag}
'
165 if version and version != tag:
166 label += f' build {version}
'
167 if channel != origin:
168 label += f' from {origin}
'
175 Update target information
177 Can be created by `query_update()` or manually.
180 tag The release tag that will be updated to. If from query_update,
181 the value is after API resolution and update spec processing.
182 The only property that is required.
183 version The actual numeric version (if available) of the binary to be updated to,
184 after API resolution and update spec processing. (default: None)
185 requested_version Numeric version of the binary being requested (if available),
186 after API resolution only. (default: None)
187 commit Commit hash (if available) of the binary to be updated to,
188 after API resolution and update spec processing. (default: None)
189 This value will only match the RELEASE_GIT_HEAD of prerelease builds.
190 binary_name Filename of the binary to be updated to. (default: current binary name)
191 checksum Expected checksum (if available) of the binary to be
192 updated to. (default: None)
195 version: str | None = None
196 requested_version: str | None = None
197 commit: str | None = None
199 binary_name: str | None = _get_binary_name()
200 checksum: str | None = None
206 # XXX: use class variables to simplify testing
210 def __init__(self, ydl, target: str | None = None):
212 # For backwards compat, target needs to be treated as if it could be None
213 self.requested_channel, sep, self.requested_tag = (target or self._channel).rpartition('@')
214 # Check if requested_tag is actually the requested repo/channel
215 if not sep and ('/' in self.requested_tag or self.requested_tag in UPDATE_SOURCES):
216 self.requested_channel = self.requested_tag
217 self.requested_tag: str = None # type: ignore (we set it later)
218 elif not self.requested_channel:
219 # User did not specify a channel, so we are requesting the default channel
220 self.requested_channel = self._channel.partition('@')[0]
222 # --update should not be treated as an exact tag request even if CHANNEL has a @tag
223 self._exact = bool(target) and target != self._channel
224 if not self.requested_tag:
225 # User did not specify a tag, so we request 'latest
' and track that no exact tag was passed
226 self.requested_tag = 'latest
'
229 if '/' in self.requested_channel:
230 # requested_channel is actually a repository
231 self.requested_repo = self.requested_channel
232 if not self.requested_repo.startswith('yt
-dlp
/') and self.requested_repo != self._origin:
233 self.ydl.report_warning(
234 f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable
'
235 f'from {self.ydl._format_err(self.requested_repo, self.ydl.Styles.EMPHASIS)}
. '
236 f'Run {self.ydl._format_err("at your own risk", "light red")}
')
237 self._block_restart('Automatically restarting into custom builds
is disabled
for security reasons
')
239 # Check if requested_channel resolves to a known repository or else raise
240 self.requested_repo = UPDATE_SOURCES.get(self.requested_channel)
241 if not self.requested_repo:
243 f'Invalid update channel {self.requested_channel!r} requested
. '
244 f'Valid channels are {", ".join(UPDATE_SOURCES)}
', True)
246 self._identifier = f'{detect_variant()} {system_identifier()}
'
249 def current_version(self):
250 """Current version"""
254 def current_commit(self):
255 """Current commit hash"""
256 return RELEASE_GIT_HEAD
258 def _download_asset(self, name, tag=None):
260 tag = self.requested_tag
262 path = 'latest
/download
' if tag == 'latest
' else f'download
/{tag}
'
263 url = f'https
://github
.com
/{self.requested_repo}
/releases
/{path}
/{name}
'
264 self.ydl.write_debug(f'Downloading {name}
from {url}
')
265 return self.ydl.urlopen(url).read()
267 def _call_api(self, tag):
268 tag = f'tags
/{tag}
' if tag != 'latest
' else tag
269 url = f'{API_BASE_URL}
/{self.requested_repo}
/releases
/{tag}
'
270 self.ydl.write_debug(f'Fetching release info
: {url}
')
271 return json.loads(self.ydl.urlopen(Request(url, headers={
272 'Accept
': 'application
/vnd
.github
+json
',
273 'User
-Agent
': 'yt
-dlp
',
274 'X
-GitHub
-Api
-Version
': '2022-11-28',
277 def _get_version_info(self, tag: str) -> tuple[str | None, str | None]:
278 if _VERSION_RE.fullmatch(tag):
281 api_info = self._call_api(tag)
284 requested_version = api_info['tag_name
']
286 match = re.search(rf'\s
+(?P
<version
>{_VERSION_RE.pattern}
)$
', api_info.get('name
', ''))
287 requested_version = match.group('version
') if match else None
289 if re.fullmatch(_HASH_PATTERN, api_info.get('target_commitish
', '')):
290 target_commitish = api_info['target_commitish
']
292 match = _COMMIT_RE.match(api_info.get('body
', ''))
293 target_commitish = match.group('hash') if match else None
295 if not (requested_version or target_commitish):
296 self._report_error('One of either version
or commit
hash must be available on the release
', expected=True)
298 return requested_version, target_commitish
300 def _download_update_spec(self, source_tags):
301 for tag in source_tags:
303 return self._download_asset('_update_spec
', tag=tag).decode()
304 except network_exceptions as error:
305 if isinstance(error, HTTPError) and error.status == 404:
307 self._report_network_error(f'fetch update spec
: {error}
')
310 f'The requested tag {self.requested_tag} does
not exist
for {self.requested_repo}
', True)
313 def _process_update_spec(self, lockfile: str, resolved_tag: str):
314 lines = lockfile.splitlines()
315 is_version2 = any(line.startswith('lockV2
') for line in lines)
319 if not line.startswith(f'lockV2 {self.requested_repo}
'):
321 _, _, tag, pattern = line.split(' ', 3)
323 if not line.startswith('lock
'):
325 _, tag, pattern = line.split(' ', 2)
327 if re.match(pattern, self._identifier):
328 if _VERSION_RE.fullmatch(tag):
331 elif self._version_compare(tag, resolved_tag):
333 elif tag != resolved_tag:
337 f'yt
-dlp cannot be updated to {resolved_tag} since you are on an older Python version
', True)
342 def _version_compare(self, a: str, b: str):
344 Compare two version strings
346 This function SHOULD NOT be called if self._exact == True
348 if _VERSION_RE.fullmatch(f'{a}
.{b}
'):
349 return version_tuple(a) >= version_tuple(b)
352 def query_update(self, *, _output=False) -> UpdateInfo | None:
353 """Fetches and returns info about the available update"""
354 if not self.requested_repo:
355 self._report_error('No target repository could be determined
from input')
359 requested_version, target_commitish = self._get_version_info(self.requested_tag)
360 except network_exceptions as e:
361 self._report_network_error(f'obtain version
info ({e}
)', delim='; Please
try again later
or')
364 if self._exact and self._origin != self.requested_repo:
366 elif requested_version:
368 has_update = self.current_version != requested_version
370 has_update = not self._version_compare(self.current_version, requested_version)
371 elif target_commitish:
372 has_update = target_commitish != self.current_commit
376 resolved_tag = requested_version if self.requested_tag == 'latest
' else self.requested_tag
377 current_label = _make_label(self._origin, self._channel.partition("@")[2] or self.current_version, self.current_version)
378 requested_label = _make_label(self.requested_repo, resolved_tag, requested_version)
379 latest_or_requested = f'{"Latest" if self.requested_tag == "latest" else "Requested"} version
: {requested_label}
'
382 self.ydl.to_screen(f'{latest_or_requested}
\nyt
-dlp
is up to
date ({current_label}
)')
385 update_spec = self._download_update_spec(('latest
', None) if requested_version else (None,))
388 # `result_` prefixed vars == post-_process_update_spec() values
389 result_tag = self._process_update_spec(update_spec, resolved_tag)
390 if not result_tag or result_tag == self.current_version:
392 elif result_tag == resolved_tag:
393 result_version = requested_version
394 elif _VERSION_RE.fullmatch(result_tag):
395 result_version = result_tag
396 else: # actual version being updated to is unknown
397 result_version = None
400 # Non-updateable variants can get update_info but need to skip checksum
401 if not is_non_updateable():
403 hashes = self._download_asset('SHA2
-256SUMS
', result_tag)
404 except network_exceptions as error:
405 if not isinstance(error, HTTPError) or error.status != 404:
406 self._report_network_error(f'fetch checksums
: {error}
')
408 self.ydl.report_warning('No
hash information found
for the release
, skipping verification
')
410 for ln in hashes.decode().splitlines():
411 if ln.endswith(_get_binary_name()):
412 checksum = ln.split()[0]
415 self.ydl.report_warning('The
hash could
not be found
in the checksum
file, skipping verification
')
418 update_label = _make_label(self.requested_repo, result_tag, result_version)
420 f'Current version
: {current_label}
\n{latest_or_requested}
'
421 + (f'\nUpgradable to
: {update_label}
' if update_label != requested_label else ''))
425 version=result_version,
426 requested_version=requested_version,
427 commit=target_commitish if result_tag == resolved_tag else None,
430 def update(self, update_info=NO_DEFAULT):
431 """Update yt-dlp executable to the latest version"""
432 if update_info is NO_DEFAULT:
433 update_info = self.query_update(_output=True)
437 err = is_non_updateable()
439 self._report_error(err, True)
442 self.ydl.to_screen(f'Current Build Hash
: {_sha256_file(self.filename)}
')
444 update_label = _make_label(self.requested_repo, update_info.tag, update_info.version)
445 self.ydl.to_screen(f'Updating to {update_label}
...')
447 directory = os.path.dirname(self.filename)
448 if not os.access(self.filename, os.W_OK):
449 return self._report_permission_error(self.filename)
450 elif not os.access(directory, os.W_OK):
451 return self._report_permission_error(directory)
453 new_filename, old_filename = f'{self.filename}
.new
', f'{self.filename}
.old
'
454 if detect_variant() == 'zip': # Can be replaced in-place
455 new_filename, old_filename = self.filename, None
458 if os.path.exists(old_filename or ''):
459 os.remove(old_filename)
461 return self._report_error('Unable to remove the old version
')
464 newcontent = self._download_asset(update_info.binary_name, update_info.tag)
465 except network_exceptions as e:
466 if isinstance(e, HTTPError) and e.status == 404:
467 return self._report_error(
468 f'The requested tag {self.requested_repo}
@{update_info.tag} does
not exist
', True)
469 return self._report_network_error(f'fetch updates
: {e}
', tag=update_info.tag)
471 if not update_info.checksum:
472 self._block_restart('Automatically restarting into unverified builds
is disabled
for security reasons
')
473 elif hashlib.sha256(newcontent).hexdigest() != update_info.checksum:
474 return self._report_network_error('verify the new executable
', tag=update_info.tag)
477 with open(new_filename, 'wb
') as outf:
478 outf.write(newcontent)
480 return self._report_permission_error(new_filename)
483 mask = os.stat(self.filename).st_mode
485 os.rename(self.filename, old_filename)
487 return self._report_error('Unable to move current version
')
490 os.rename(new_filename, self.filename)
492 self._report_error('Unable to overwrite current version
')
493 return os.rename(old_filename, self.filename)
495 variant = detect_variant()
496 if variant.startswith('win
') or variant == 'py2exe
':
497 atexit.register(Popen, f'ping
127.0.0.1 -n
5 -w
1000 & del /F
"{old_filename}"',
498 shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
501 os.remove(old_filename)
503 self._report_error('Unable to remove the old version
')
506 os.chmod(self.filename, mask)
508 return self._report_error(
509 f'Unable to
set permissions
. Run
: sudo chmod a
+rx {compat_shlex_quote(self.filename)}
')
511 self.ydl.to_screen(f'Updated yt
-dlp to {update_label}
')
514 @functools.cached_property
516 """Filename of the executable"""
517 return compat_realpath(_get_variant_and_executable_path()[1])
519 @functools.cached_property
521 """The command-line to run the executable, if known"""
522 # There is no sys.orig_argv in py < 3.10. Also, it can be [] when frozen
523 if getattr(sys, 'orig_argv
', None):
525 elif getattr(sys, 'frozen
', False):
529 """Restart the executable"""
530 assert self.cmd, 'Must be frozen
or Py
>= 3.10'
531 self.ydl.write_debug(f'Restarting
: {shell_quote(self.cmd)}
')
532 _, _, returncode = Popen.run(self.cmd)
535 def _block_restart(self, msg):
537 self._report_error(f'{msg}
. Restart yt
-dlp to use the updated version
', expected=True)
538 return self.ydl._download_retcode
539 self.restart = wrapper
541 def _report_error(self, msg, expected=False):
542 self.ydl.report_error(msg, tb=False if expected else None)
543 self.ydl._download_retcode = 100
545 def _report_permission_error(self, file):
546 self._report_error(f'Unable to write to {file}
; try running
as administrator
', True)
548 def _report_network_error(self, action, delim=';', tag=None):
550 tag = self.requested_tag
552 f'Unable to {action}{delim} visit https
://github
.com
/{self.requested_repo}
/releases
/'
553 + tag if tag == "latest" else f"tag/{tag}", True)
555 # XXX: Everything below this line in this class is deprecated / for compat only
557 def _target_tag(self):
558 """Deprecated; requested tag with 'tags
/' prepended when necessary for API calls"""
559 return f'tags
/{self.requested_tag}
' if self.requested_tag != 'latest
' else self.requested_tag
561 def _check_update(self):
562 """Deprecated; report whether there is an update available"""
563 return bool(self.query_update(_output=True))
565 def __getattr__(self, attribute: str):
566 """Compat getter function for deprecated attributes"""
567 deprecated_props_map = {
568 'check_update
': '_check_update
',
569 'target_tag
': '_target_tag
',
570 'target_channel
': 'requested_channel
',
572 update_info_props_map = {
573 'has_update
': '_has_update
',
574 'new_version
': 'version
',
575 'latest_version
': 'requested_version
',
576 'release_name
': 'binary_name
',
577 'release_hash
': 'checksum
',
580 if attribute not in deprecated_props_map and attribute not in update_info_props_map:
581 raise AttributeError(f'{type(self).__name__!r}
object has no attribute {attribute!r}
')
583 msg = f'{type(self).__name__}
.{attribute}
is deprecated
and will be removed
in a future version
'
584 if attribute in deprecated_props_map:
585 source_name = deprecated_props_map[attribute]
586 if not source_name.startswith('_
'):
587 msg += f'. Please use {source_name!r} instead
'
589 mapping = deprecated_props_map
591 else: # attribute in update_info_props_map
592 msg += '. Please call
query_update() instead
'
593 source = self.query_update()
595 source = UpdateInfo('', None, None, None)
596 source._has_update = False
597 mapping = update_info_props_map
599 deprecation_warning(msg)
600 for target_name, source_name in mapping.items():
601 value = getattr(source, source_name)
602 setattr(self, target_name, value)
604 return getattr(self, attribute)
608 """Update the program file with the latest version from the repository
609 @returns Whether there was a successful update (No update = False)
611 return Updater(ydl).update()
614 __all__ = ['Updater
']