import base64
import collections
import contextlib
+import datetime as dt
+import glob
import http.cookiejar
import http.cookies
import io
import tempfile
import time
import urllib.request
-from datetime import datetime, timedelta, timezone
from enum import Enum, auto
from hashlib import pbkdf2_hmac
aes_gcm_decrypt_and_verify_bytes,
unpad_pkcs7,
)
-from .compat import functools
+from .compat import functools # isort: split
+from .compat import compat_os_name
from .dependencies import (
_SECRETSTORAGE_UNAVAILABLE_REASON,
secretstorage,
)
from .minicurses import MultilinePrinter, QuietMultilinePrinter
from .utils import (
+ DownloadError,
Popen,
error_to_str,
- escape_url,
expand_path,
is_path_like,
sanitize_url,
try_call,
write_string,
)
+from .utils._utils import _YDLLogger
+from .utils.networking import normalize_url
CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
-class YDLLogger:
- def __init__(self, ydl=None):
- self._ydl = ydl
-
- def debug(self, message):
- if self._ydl:
- self._ydl.write_debug(message)
-
- def info(self, message):
- if self._ydl:
- self._ydl.to_screen(f'[Cookies] {message}')
-
- def warning(self, message, only_once=False):
- if self._ydl:
- self._ydl.report_warning(message, only_once)
-
- def error(self, message):
- if self._ydl:
- self._ydl.report_error(message)
+class YDLLogger(_YDLLogger):
+ def warning(self, message, only_once=False): # compat
+ return super().warning(message, once=only_once)
class ProgressBar(MultilinePrinter):
_DELAY, _timer = 0.1, 0
jar = YoutubeDLCookieJar(cookie_file)
if not is_filename or os.access(cookie_file, os.R_OK):
- jar.load(ignore_discard=True, ignore_expires=True)
+ jar.load()
cookie_jars.append(jar)
return _merge_cookie_jars(cookie_jars)
logger.info('Extracting cookies from firefox')
if not sqlite3:
logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
- 'Please use a python interpreter compiled with sqlite3 support')
+ 'Please use a Python interpreter compiled with sqlite3 support')
return YoutubeDLCookieJar()
if profile is None:
- search_root = _firefox_browser_dir()
+ search_roots = list(_firefox_browser_dirs())
elif _is_path(profile):
- search_root = profile
+ search_roots = [profile]
else:
- search_root = os.path.join(_firefox_browser_dir(), profile)
+ search_roots = [os.path.join(path, profile) for path in _firefox_browser_dirs()]
+ search_root = ', '.join(map(repr, search_roots))
- cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger)
+ cookie_database_path = _newest(_firefox_cookie_dbs(search_roots))
if cookie_database_path is None:
raise FileNotFoundError(f'could not find firefox cookies database in {search_root}')
logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
containers_path = os.path.join(os.path.dirname(cookie_database_path), 'containers.json')
if not os.path.isfile(containers_path) or not os.access(containers_path, os.R_OK):
raise FileNotFoundError(f'could not read containers.json in {search_root}')
- with open(containers_path) as containers:
+ with open(containers_path, encoding='utf8') as containers:
identities = json.load(containers).get('identities', [])
container_id = next((context.get('userContextId') for context in identities if container in (
context.get('name'),
cursor.connection.close()
-def _firefox_browser_dir():
+def _firefox_browser_dirs():
if sys.platform in ('cygwin', 'win32'):
- return os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
+ yield os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
+
elif sys.platform == 'darwin':
- return os.path.expanduser('~/Library/Application Support/Firefox')
- return os.path.expanduser('~/.mozilla/firefox')
+ yield os.path.expanduser('~/Library/Application Support/Firefox/Profiles')
+
+ else:
+ yield from map(os.path.expanduser, ('~/.mozilla/firefox', '~/snap/firefox/common/.mozilla/firefox'))
+
+
+def _firefox_cookie_dbs(roots):
+ for root in map(os.path.abspath, roots):
+ for pattern in ('', '*/', 'Profiles/*/'):
+ yield from glob.iglob(os.path.join(root, pattern, 'cookies.sqlite'))
def _get_chromium_based_browser_settings(browser_name):
if not sqlite3:
logger.warning(f'Cannot extract cookies from {browser_name} without sqlite3 support. '
- 'Please use a python interpreter compiled with sqlite3 support')
+ 'Please use a Python interpreter compiled with sqlite3 support')
return YoutubeDLCookieJar()
config = _get_chromium_based_browser_settings(browser_name)
logger.error(f'{browser_name} does not support profiles')
search_root = config['browser_dir']
- cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies', logger)
+ cookie_database_path = _newest(_find_files(search_root, 'Cookies', logger))
if cookie_database_path is None:
raise FileNotFoundError(f'could not find {browser_name} cookies database in "{search_root}"')
logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
counts['unencrypted'] = unencrypted_cookies
logger.debug(f'cookie version breakdown: {counts}')
return jar
+ except PermissionError as error:
+ if compat_os_name == 'nt' and error.errno == 13:
+ message = 'Could not copy Chrome cookie database. See https://github.com/yt-dlp/yt-dlp/issues/7271 for more info'
+ logger.error(message)
+ raise DownloadError(message) # force exit
+ raise
finally:
if cursor is not None:
cursor.connection.close()
def _mac_absolute_time_to_posix(timestamp):
- return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp())
+ return int((dt.datetime(2001, 1, 1, 0, 0, tzinfo=dt.timezone.utc) + dt.timedelta(seconds=timestamp)).timestamp())
def _parse_safari_cookies_header(data, logger):
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_util_linux.h
SelectedLinuxBackend
"""
- KWALLET4 = auto() # this value is just called KWALLET in the chromium source but it is for KDE4 only
+ KWALLET = auto() # KDE4
KWALLET5 = auto()
KWALLET6 = auto()
- GNOME_KEYRING = auto()
- BASIC_TEXT = auto()
+ GNOMEKEYRING = auto()
+ BASICTEXT = auto()
SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
desktop_environment = _get_linux_desktop_environment(os.environ, logger)
logger.debug(f'detected desktop environment: {desktop_environment.name}')
if desktop_environment == _LinuxDesktopEnvironment.KDE4:
- linux_keyring = _LinuxKeyring.KWALLET4
+ linux_keyring = _LinuxKeyring.KWALLET
elif desktop_environment == _LinuxDesktopEnvironment.KDE5:
linux_keyring = _LinuxKeyring.KWALLET5
elif desktop_environment == _LinuxDesktopEnvironment.KDE6:
elif desktop_environment in (
_LinuxDesktopEnvironment.KDE3, _LinuxDesktopEnvironment.LXQT, _LinuxDesktopEnvironment.OTHER
):
- linux_keyring = _LinuxKeyring.BASIC_TEXT
+ linux_keyring = _LinuxKeyring.BASICTEXT
else:
- linux_keyring = _LinuxKeyring.GNOME_KEYRING
+ linux_keyring = _LinuxKeyring.GNOMEKEYRING
return linux_keyring
"""
default_wallet = 'kdewallet'
try:
- if keyring == _LinuxKeyring.KWALLET4:
+ if keyring == _LinuxKeyring.KWALLET:
service_name = 'org.kde.kwalletd'
wallet_path = '/modules/kwalletd'
elif keyring == _LinuxKeyring.KWALLET5:
keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
logger.debug(f'Chosen keyring: {keyring.name}')
- if keyring in (_LinuxKeyring.KWALLET4, _LinuxKeyring.KWALLET5, _LinuxKeyring.KWALLET6):
+ if keyring in (_LinuxKeyring.KWALLET, _LinuxKeyring.KWALLET5, _LinuxKeyring.KWALLET6):
return _get_kwallet_password(browser_keyring_name, keyring, logger)
- elif keyring == _LinuxKeyring.GNOME_KEYRING:
+ elif keyring == _LinuxKeyring.GNOMEKEYRING:
return _get_gnome_keyring_password(browser_keyring_name, logger)
- elif keyring == _LinuxKeyring.BASIC_TEXT:
+ elif keyring == _LinuxKeyring.BASICTEXT:
# when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
return None
assert False, f'Unknown keyring {keyring}'
References:
- [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
"""
- path = _find_most_recently_used_file(browser_root, 'Local State', logger)
+ path = _newest(_find_files(browser_root, 'Local State', logger))
if path is None:
logger.error('could not find local state file')
return None
return [row[1].decode() for row in table_info]
-def _find_most_recently_used_file(root, filename, logger):
+def _newest(files):
+ return max(files, key=lambda path: os.lstat(path).st_mtime, default=None)
+
+
+def _find_files(root, filename, logger):
# if there are multiple browser profiles, take the most recently used one
- i, paths = 0, []
+ i = 0
with _create_progress_bar(logger) as progress_bar:
- for curr_root, dirs, files in os.walk(root):
+ for curr_root, _, files in os.walk(root):
for file in files:
i += 1
progress_bar.print(f'Searching for "{filename}": {i: 6d} files searched')
if file == filename:
- paths.append(os.path.join(curr_root, file))
- return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
+ yield os.path.join(curr_root, file)
def _merge_cookie_jars(jars):
def _is_path(value):
- return os.path.sep in value
+ return any(sep in value for sep in (os.path.sep, os.path.altsep) if sep)
def _parse_browser_specification(browser_name, profile=None, keyring=None, container=None):
file.truncate(0)
yield file
- def _really_save(self, f, ignore_discard=False, ignore_expires=False):
+ def _really_save(self, f, ignore_discard, ignore_expires):
now = time.time()
for cookie in self:
if (not ignore_discard and cookie.discard
name, value
)))
- def save(self, filename=None, *args, **kwargs):
+ def save(self, filename=None, ignore_discard=True, ignore_expires=True):
"""
Save cookies to a file.
Code is taken from CPython 3.6
with self.open(filename, write=True) as f:
f.write(self._HEADER)
- self._really_save(f, *args, **kwargs)
+ self._really_save(f, ignore_discard, ignore_expires)
- def load(self, filename=None, ignore_discard=False, ignore_expires=False):
+ def load(self, filename=None, ignore_discard=True, ignore_expires=True):
"""Load cookies from a file."""
if filename is None:
if self.filename is not None:
def get_cookie_header(self, url):
"""Generate a Cookie HTTP header for a given url"""
- cookie_req = urllib.request.Request(escape_url(sanitize_url(url)))
+ cookie_req = urllib.request.Request(normalize_url(sanitize_url(url)))
self.add_cookie_header(cookie_req)
return cookie_req.get_header('Cookie')
+
+ def get_cookies_for_url(self, url):
+ """Generate a list of Cookie objects for a given url"""
+ # Policy `_now` attribute must be set before calling `_cookies_for_request`
+ # Ref: https://github.com/python/cpython/blob/3.7/Lib/http/cookiejar.py#L1360
+ self._policy._now = self._now = int(time.time())
+ return self._cookies_for_request(urllib.request.Request(normalize_url(sanitize_url(url))))
+
+ def clear(self, *args, **kwargs):
+ with contextlib.suppress(KeyError):
+ return super().clear(*args, **kwargs)