]> jfr.im git - yt-dlp.git/blob - yt_dlp/update.py
85c9bb96226f3f2bff0041993a1356160c2aa1d7
[yt-dlp.git] / yt_dlp / update.py
1 from __future__ import annotations
2
3 import atexit
4 import contextlib
5 import hashlib
6 import json
7 import os
8 import platform
9 import re
10 import subprocess
11 import sys
12 from dataclasses import dataclass
13 from zipimport import zipimporter
14
15 from .compat import functools # isort: split
16 from .compat import compat_realpath, compat_shlex_quote
17 from .networking import Request
18 from .networking.exceptions import HTTPError, network_exceptions
19 from .utils import (
20 NO_DEFAULT,
21 Popen,
22 deprecation_warning,
23 format_field,
24 remove_end,
25 shell_quote,
26 system_identifier,
27 version_tuple,
28 )
29 from .version import (
30 CHANNEL,
31 ORIGIN,
32 RELEASE_GIT_HEAD,
33 UPDATE_HINT,
34 VARIANT,
35 __version__,
36 )
37
38 UPDATE_SOURCES = {
39 'stable': 'yt-dlp/yt-dlp',
40 'nightly': 'yt-dlp/yt-dlp-nightly-builds',
41 'master': 'yt-dlp/yt-dlp-master-builds',
42 }
43 REPOSITORY = UPDATE_SOURCES['stable']
44 _INVERSE_UPDATE_SOURCES = {value: key for key, value in UPDATE_SOURCES.items()}
45
46 _VERSION_RE = re.compile(r'(\d+\.)*\d+')
47 _HASH_PATTERN = r'[\da-f]{40}'
48 _COMMIT_RE = re.compile(rf'Generated from: https://(?:[^/?#]+/){{3}}commit/(?P<hash>{_HASH_PATTERN})')
49
50 API_BASE_URL = 'https://api.github.com/repos'
51
52 # Backwards compatibility variables for the current channel
53 API_URL = f'{API_BASE_URL}/{REPOSITORY}/releases'
54
55
56 @functools.cache
57 def _get_variant_and_executable_path():
58 """@returns (variant, executable_path)"""
59 if getattr(sys, 'frozen', False):
60 path = sys.executable
61 if not hasattr(sys, '_MEIPASS'):
62 return 'py2exe', path
63 elif sys._MEIPASS == os.path.dirname(path):
64 return f'{sys.platform}_dir', path
65 elif sys.platform == 'darwin':
66 machine = '_legacy' if version_tuple(platform.mac_ver()[0]) < (10, 15) else ''
67 else:
68 machine = f'_{platform.machine().lower()}'
69 # Ref: https://en.wikipedia.org/wiki/Uname#Examples
70 if machine[1:] in ('x86', 'x86_64', 'amd64', 'i386', 'i686'):
71 machine = '_x86' if platform.architecture()[0][:2] == '32' else ''
72 return f'{remove_end(sys.platform, "32")}{machine}_exe', path
73
74 path = os.path.dirname(__file__)
75 if isinstance(__loader__, zipimporter):
76 return 'zip', os.path.join(path, '..')
77 elif (os.path.basename(sys.argv[0]) in ('__main__.py', '-m')
78 and os.path.exists(os.path.join(path, '../.git/HEAD'))):
79 return 'source', path
80 return 'unknown', path
81
82
83 def detect_variant():
84 return VARIANT or _get_variant_and_executable_path()[0]
85
86
87 @functools.cache
88 def current_git_head():
89 if detect_variant() != 'source':
90 return
91 with contextlib.suppress(Exception):
92 stdout, _, _ = Popen.run(
93 ['git', 'rev-parse', '--short', 'HEAD'],
94 text=True, cwd=os.path.dirname(os.path.abspath(__file__)),
95 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
96 if re.fullmatch('[0-9a-f]+', stdout.strip()):
97 return stdout.strip()
98
99
100 _FILE_SUFFIXES = {
101 'zip': '',
102 'py2exe': '_min.exe',
103 'win_exe': '.exe',
104 'win_x86_exe': '_x86.exe',
105 'darwin_exe': '_macos',
106 'darwin_legacy_exe': '_macos_legacy',
107 'linux_exe': '_linux',
108 'linux_aarch64_exe': '_linux_aarch64',
109 'linux_armv7l_exe': '_linux_armv7l',
110 }
111
112 _NON_UPDATEABLE_REASONS = {
113 **{variant: None for variant in _FILE_SUFFIXES}, # Updatable
114 **{variant: f'Auto-update is not supported for unpackaged {name} executable; Re-download the latest release'
115 for variant, name in {'win32_dir': 'Windows', 'darwin_dir': 'MacOS', 'linux_dir': 'Linux'}.items()},
116 'source': 'You cannot update when running from source code; Use git to pull the latest changes',
117 'unknown': 'You installed yt-dlp with a package manager or setup.py; Use that to update',
118 'other': 'You are using an unofficial build of yt-dlp; Build the executable again',
119 }
120
121
122 def is_non_updateable():
123 if UPDATE_HINT:
124 return UPDATE_HINT
125 return _NON_UPDATEABLE_REASONS.get(
126 detect_variant(), _NON_UPDATEABLE_REASONS['unknown' if VARIANT else 'other'])
127
128
129 def _get_binary_name():
130 return format_field(_FILE_SUFFIXES, detect_variant(), template='yt-dlp%s', ignore=None, default=None)
131
132
133 def _get_system_deprecation():
134 MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 8)
135
136 if sys.version_info > MIN_RECOMMENDED:
137 return None
138
139 major, minor = sys.version_info[:2]
140 if sys.version_info < MIN_SUPPORTED:
141 msg = f'Python version {major}.{minor} is no longer supported'
142 else:
143 msg = f'Support for Python version {major}.{minor} has been deprecated. '
144 # Temporary until `win_x86_exe` uses 3.8, which will deprecate Vista and Server 2008
145 if detect_variant() == 'win_x86_exe':
146 platform_name = platform.platform()
147 if any(platform_name.startswith(f'Windows-{name}') for name in ('Vista', '2008Server')):
148 msg = 'Support for Windows Vista/Server 2008 has been deprecated. '
149 else:
150 return None
151 msg += ('See https://github.com/yt-dlp/yt-dlp/issues/7803 for details.'
152 '\nYou may stop receiving updates on this version at any time')
153
154 major, minor = MIN_RECOMMENDED
155 return f'{msg}! Please update to Python {major}.{minor} or above'
156
157
158 def _sha256_file(path):
159 h = hashlib.sha256()
160 mv = memoryview(bytearray(128 * 1024))
161 with open(os.path.realpath(path), 'rb', buffering=0) as f:
162 for n in iter(lambda: f.readinto(mv), 0):
163 h.update(mv[:n])
164 return h.hexdigest()
165
166
167 def _make_label(origin, tag, version=None):
168 if '/' in origin:
169 channel = _INVERSE_UPDATE_SOURCES.get(origin, origin)
170 else:
171 channel = origin
172 label = f'{channel}@{tag}'
173 if version and version != tag:
174 label += f' build {version}'
175 if channel != origin:
176 label += f' from {origin}'
177 return label
178
179
180 @dataclass
181 class UpdateInfo:
182 """
183 Update target information
184
185 Can be created by `query_update()` or manually.
186
187 Attributes:
188 tag The release tag that will be updated to. If from query_update,
189 the value is after API resolution and update spec processing.
190 The only property that is required.
191 version The actual numeric version (if available) of the binary to be updated to,
192 after API resolution and update spec processing. (default: None)
193 requested_version Numeric version of the binary being requested (if available),
194 after API resolution only. (default: None)
195 commit Commit hash (if available) of the binary to be updated to,
196 after API resolution and update spec processing. (default: None)
197 This value will only match the RELEASE_GIT_HEAD of prerelease builds.
198 binary_name Filename of the binary to be updated to. (default: current binary name)
199 checksum Expected checksum (if available) of the binary to be
200 updated to. (default: None)
201 """
202 tag: str
203 version: str | None = None
204 requested_version: str | None = None
205 commit: str | None = None
206
207 binary_name: str | None = _get_binary_name()
208 checksum: str | None = None
209
210 _has_update = True
211
212
213 class Updater:
214 # XXX: use class variables to simplify testing
215 _channel = CHANNEL
216 _origin = ORIGIN
217
218 def __init__(self, ydl, target: str | None = None):
219 self.ydl = ydl
220 # For backwards compat, target needs to be treated as if it could be None
221 self.requested_channel, sep, self.requested_tag = (target or self._channel).rpartition('@')
222 # Check if requested_tag is actually the requested repo/channel
223 if not sep and ('/' in self.requested_tag or self.requested_tag in UPDATE_SOURCES):
224 self.requested_channel = self.requested_tag
225 self.requested_tag: str = None # type: ignore (we set it later)
226 elif not self.requested_channel:
227 # User did not specify a channel, so we are requesting the default channel
228 self.requested_channel = self._channel.partition('@')[0]
229
230 # --update should not be treated as an exact tag request even if CHANNEL has a @tag
231 self._exact = bool(target) and target != self._channel
232 if not self.requested_tag:
233 # User did not specify a tag, so we request 'latest' and track that no exact tag was passed
234 self.requested_tag = 'latest'
235 self._exact = False
236
237 if '/' in self.requested_channel:
238 # requested_channel is actually a repository
239 self.requested_repo = self.requested_channel
240 if not self.requested_repo.startswith('yt-dlp/') and self.requested_repo != self._origin:
241 self.ydl.report_warning(
242 f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable '
243 f'from {self.ydl._format_err(self.requested_repo, self.ydl.Styles.EMPHASIS)}. '
244 f'Run {self.ydl._format_err("at your own risk", "light red")}')
245 self._block_restart('Automatically restarting into custom builds is disabled for security reasons')
246 else:
247 # Check if requested_channel resolves to a known repository or else raise
248 self.requested_repo = UPDATE_SOURCES.get(self.requested_channel)
249 if not self.requested_repo:
250 self._report_error(
251 f'Invalid update channel {self.requested_channel!r} requested. '
252 f'Valid channels are {", ".join(UPDATE_SOURCES)}', True)
253
254 self._identifier = f'{detect_variant()} {system_identifier()}'
255
256 @property
257 def current_version(self):
258 """Current version"""
259 return __version__
260
261 @property
262 def current_commit(self):
263 """Current commit hash"""
264 return RELEASE_GIT_HEAD
265
266 def _download_asset(self, name, tag=None):
267 if not tag:
268 tag = self.requested_tag
269
270 path = 'latest/download' if tag == 'latest' else f'download/{tag}'
271 url = f'https://github.com/{self.requested_repo}/releases/{path}/{name}'
272 self.ydl.write_debug(f'Downloading {name} from {url}')
273 return self.ydl.urlopen(url).read()
274
275 def _call_api(self, tag):
276 tag = f'tags/{tag}' if tag != 'latest' else tag
277 url = f'{API_BASE_URL}/{self.requested_repo}/releases/{tag}'
278 self.ydl.write_debug(f'Fetching release info: {url}')
279 return json.loads(self.ydl.urlopen(Request(url, headers={
280 'Accept': 'application/vnd.github+json',
281 'User-Agent': 'yt-dlp',
282 'X-GitHub-Api-Version': '2022-11-28',
283 })).read().decode())
284
285 def _get_version_info(self, tag: str) -> tuple[str | None, str | None]:
286 if _VERSION_RE.fullmatch(tag):
287 return tag, None
288
289 api_info = self._call_api(tag)
290
291 if tag == 'latest':
292 requested_version = api_info['tag_name']
293 else:
294 match = re.search(rf'\s+(?P<version>{_VERSION_RE.pattern})$', api_info.get('name', ''))
295 requested_version = match.group('version') if match else None
296
297 if re.fullmatch(_HASH_PATTERN, api_info.get('target_commitish', '')):
298 target_commitish = api_info['target_commitish']
299 else:
300 match = _COMMIT_RE.match(api_info.get('body', ''))
301 target_commitish = match.group('hash') if match else None
302
303 if not (requested_version or target_commitish):
304 self._report_error('One of either version or commit hash must be available on the release', expected=True)
305
306 return requested_version, target_commitish
307
308 def _download_update_spec(self, source_tags):
309 for tag in source_tags:
310 try:
311 return self._download_asset('_update_spec', tag=tag).decode()
312 except network_exceptions as error:
313 if isinstance(error, HTTPError) and error.status == 404:
314 continue
315 self._report_network_error(f'fetch update spec: {error}')
316
317 self._report_error(
318 f'The requested tag {self.requested_tag} does not exist for {self.requested_repo}', True)
319 return None
320
321 def _process_update_spec(self, lockfile: str, resolved_tag: str):
322 lines = lockfile.splitlines()
323 is_version2 = any(line.startswith('lockV2 ') for line in lines)
324
325 for line in lines:
326 if is_version2:
327 if not line.startswith(f'lockV2 {self.requested_repo} '):
328 continue
329 _, _, tag, pattern = line.split(' ', 3)
330 else:
331 if not line.startswith('lock '):
332 continue
333 _, tag, pattern = line.split(' ', 2)
334
335 if re.match(pattern, self._identifier):
336 if _VERSION_RE.fullmatch(tag):
337 if not self._exact:
338 return tag
339 elif self._version_compare(tag, resolved_tag):
340 return resolved_tag
341 elif tag != resolved_tag:
342 continue
343
344 self._report_error(
345 f'yt-dlp cannot be updated to {resolved_tag} since you are on an older Python version', True)
346 return None
347
348 return resolved_tag
349
350 def _version_compare(self, a: str, b: str):
351 """
352 Compare two version strings
353
354 This function SHOULD NOT be called if self._exact == True
355 """
356 if _VERSION_RE.fullmatch(f'{a}.{b}'):
357 return version_tuple(a) >= version_tuple(b)
358 return a == b
359
360 def query_update(self, *, _output=False) -> UpdateInfo | None:
361 """Fetches and returns info about the available update"""
362 if not self.requested_repo:
363 self._report_error('No target repository could be determined from input')
364 return None
365
366 try:
367 requested_version, target_commitish = self._get_version_info(self.requested_tag)
368 except network_exceptions as e:
369 self._report_network_error(f'obtain version info ({e})', delim='; Please try again later or')
370 return None
371
372 if self._exact and self._origin != self.requested_repo:
373 has_update = True
374 elif requested_version:
375 if self._exact:
376 has_update = self.current_version != requested_version
377 else:
378 has_update = not self._version_compare(self.current_version, requested_version)
379 elif target_commitish:
380 has_update = target_commitish != self.current_commit
381 else:
382 has_update = False
383
384 resolved_tag = requested_version if self.requested_tag == 'latest' else self.requested_tag
385 current_label = _make_label(self._origin, self._channel.partition("@")[2] or self.current_version, self.current_version)
386 requested_label = _make_label(self.requested_repo, resolved_tag, requested_version)
387 latest_or_requested = f'{"Latest" if self.requested_tag == "latest" else "Requested"} version: {requested_label}'
388 if not has_update:
389 if _output:
390 self.ydl.to_screen(f'{latest_or_requested}\nyt-dlp is up to date ({current_label})')
391 return None
392
393 update_spec = self._download_update_spec(('latest', None) if requested_version else (None,))
394 if not update_spec:
395 return None
396 # `result_` prefixed vars == post-_process_update_spec() values
397 result_tag = self._process_update_spec(update_spec, resolved_tag)
398 if not result_tag or result_tag == self.current_version:
399 return None
400 elif result_tag == resolved_tag:
401 result_version = requested_version
402 elif _VERSION_RE.fullmatch(result_tag):
403 result_version = result_tag
404 else: # actual version being updated to is unknown
405 result_version = None
406
407 checksum = None
408 # Non-updateable variants can get update_info but need to skip checksum
409 if not is_non_updateable():
410 try:
411 hashes = self._download_asset('SHA2-256SUMS', result_tag)
412 except network_exceptions as error:
413 if not isinstance(error, HTTPError) or error.status != 404:
414 self._report_network_error(f'fetch checksums: {error}')
415 return None
416 self.ydl.report_warning('No hash information found for the release, skipping verification')
417 else:
418 for ln in hashes.decode().splitlines():
419 if ln.endswith(_get_binary_name()):
420 checksum = ln.split()[0]
421 break
422 if not checksum:
423 self.ydl.report_warning('The hash could not be found in the checksum file, skipping verification')
424
425 if _output:
426 update_label = _make_label(self.requested_repo, result_tag, result_version)
427 self.ydl.to_screen(
428 f'Current version: {current_label}\n{latest_or_requested}'
429 + (f'\nUpgradable to: {update_label}' if update_label != requested_label else ''))
430
431 return UpdateInfo(
432 tag=result_tag,
433 version=result_version,
434 requested_version=requested_version,
435 commit=target_commitish if result_tag == resolved_tag else None,
436 checksum=checksum)
437
438 def update(self, update_info=NO_DEFAULT):
439 """Update yt-dlp executable to the latest version"""
440 if update_info is NO_DEFAULT:
441 update_info = self.query_update(_output=True)
442 if not update_info:
443 return False
444
445 err = is_non_updateable()
446 if err:
447 self._report_error(err, True)
448 return False
449
450 self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
451
452 update_label = _make_label(self.requested_repo, update_info.tag, update_info.version)
453 self.ydl.to_screen(f'Updating to {update_label} ...')
454
455 directory = os.path.dirname(self.filename)
456 if not os.access(self.filename, os.W_OK):
457 return self._report_permission_error(self.filename)
458 elif not os.access(directory, os.W_OK):
459 return self._report_permission_error(directory)
460
461 new_filename, old_filename = f'{self.filename}.new', f'{self.filename}.old'
462 if detect_variant() == 'zip': # Can be replaced in-place
463 new_filename, old_filename = self.filename, None
464
465 try:
466 if os.path.exists(old_filename or ''):
467 os.remove(old_filename)
468 except OSError:
469 return self._report_error('Unable to remove the old version')
470
471 try:
472 newcontent = self._download_asset(update_info.binary_name, update_info.tag)
473 except network_exceptions as e:
474 if isinstance(e, HTTPError) and e.status == 404:
475 return self._report_error(
476 f'The requested tag {self.requested_repo}@{update_info.tag} does not exist', True)
477 return self._report_network_error(f'fetch updates: {e}', tag=update_info.tag)
478
479 if not update_info.checksum:
480 self._block_restart('Automatically restarting into unverified builds is disabled for security reasons')
481 elif hashlib.sha256(newcontent).hexdigest() != update_info.checksum:
482 return self._report_network_error('verify the new executable', tag=update_info.tag)
483
484 try:
485 with open(new_filename, 'wb') as outf:
486 outf.write(newcontent)
487 except OSError:
488 return self._report_permission_error(new_filename)
489
490 if old_filename:
491 mask = os.stat(self.filename).st_mode
492 try:
493 os.rename(self.filename, old_filename)
494 except OSError:
495 return self._report_error('Unable to move current version')
496
497 try:
498 os.rename(new_filename, self.filename)
499 except OSError:
500 self._report_error('Unable to overwrite current version')
501 return os.rename(old_filename, self.filename)
502
503 variant = detect_variant()
504 if variant.startswith('win') or variant == 'py2exe':
505 atexit.register(Popen, f'ping 127.0.0.1 -n 5 -w 1000 & del /F "{old_filename}"',
506 shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
507 elif old_filename:
508 try:
509 os.remove(old_filename)
510 except OSError:
511 self._report_error('Unable to remove the old version')
512
513 try:
514 os.chmod(self.filename, mask)
515 except OSError:
516 return self._report_error(
517 f'Unable to set permissions. Run: sudo chmod a+rx {compat_shlex_quote(self.filename)}')
518
519 self.ydl.to_screen(f'Updated yt-dlp to {update_label}')
520 return True
521
522 @functools.cached_property
523 def filename(self):
524 """Filename of the executable"""
525 return compat_realpath(_get_variant_and_executable_path()[1])
526
527 @functools.cached_property
528 def cmd(self):
529 """The command-line to run the executable, if known"""
530 # There is no sys.orig_argv in py < 3.10. Also, it can be [] when frozen
531 if getattr(sys, 'orig_argv', None):
532 return sys.orig_argv
533 elif getattr(sys, 'frozen', False):
534 return sys.argv
535
536 def restart(self):
537 """Restart the executable"""
538 assert self.cmd, 'Must be frozen or Py >= 3.10'
539 self.ydl.write_debug(f'Restarting: {shell_quote(self.cmd)}')
540 _, _, returncode = Popen.run(self.cmd)
541 return returncode
542
543 def _block_restart(self, msg):
544 def wrapper():
545 self._report_error(f'{msg}. Restart yt-dlp to use the updated version', expected=True)
546 return self.ydl._download_retcode
547 self.restart = wrapper
548
549 def _report_error(self, msg, expected=False):
550 self.ydl.report_error(msg, tb=False if expected else None)
551 self.ydl._download_retcode = 100
552
553 def _report_permission_error(self, file):
554 self._report_error(f'Unable to write to {file}; try running as administrator', True)
555
556 def _report_network_error(self, action, delim=';', tag=None):
557 if not tag:
558 tag = self.requested_tag
559 self._report_error(
560 f'Unable to {action}{delim} visit https://github.com/{self.requested_repo}/releases/'
561 + tag if tag == "latest" else f"tag/{tag}", True)
562
563 # XXX: Everything below this line in this class is deprecated / for compat only
564 @property
565 def _target_tag(self):
566 """Deprecated; requested tag with 'tags/' prepended when necessary for API calls"""
567 return f'tags/{self.requested_tag}' if self.requested_tag != 'latest' else self.requested_tag
568
569 def _check_update(self):
570 """Deprecated; report whether there is an update available"""
571 return bool(self.query_update(_output=True))
572
573 def __getattr__(self, attribute: str):
574 """Compat getter function for deprecated attributes"""
575 deprecated_props_map = {
576 'check_update': '_check_update',
577 'target_tag': '_target_tag',
578 'target_channel': 'requested_channel',
579 }
580 update_info_props_map = {
581 'has_update': '_has_update',
582 'new_version': 'version',
583 'latest_version': 'requested_version',
584 'release_name': 'binary_name',
585 'release_hash': 'checksum',
586 }
587
588 if attribute not in deprecated_props_map and attribute not in update_info_props_map:
589 raise AttributeError(f'{type(self).__name__!r} object has no attribute {attribute!r}')
590
591 msg = f'{type(self).__name__}.{attribute} is deprecated and will be removed in a future version'
592 if attribute in deprecated_props_map:
593 source_name = deprecated_props_map[attribute]
594 if not source_name.startswith('_'):
595 msg += f'. Please use {source_name!r} instead'
596 source = self
597 mapping = deprecated_props_map
598
599 else: # attribute in update_info_props_map
600 msg += '. Please call query_update() instead'
601 source = self.query_update()
602 if source is None:
603 source = UpdateInfo('', None, None, None)
604 source._has_update = False
605 mapping = update_info_props_map
606
607 deprecation_warning(msg)
608 for target_name, source_name in mapping.items():
609 value = getattr(source, source_name)
610 setattr(self, target_name, value)
611
612 return getattr(self, attribute)
613
614
615 def run_update(ydl):
616 """Update the program file with the latest version from the repository
617 @returns Whether there was a successful update (No update = False)
618 """
619 return Updater(ydl).update()
620
621
622 __all__ = ['Updater']