yt_dlp/cookies.py

   1 import contextlib
   2 import ctypes
   3 import json
   4 import os
   5 import shutil
   6 import struct
   7 import subprocess
   8 import sys
   9 import tempfile
  10 from datetime import datetime, timedelta, timezone
  11 from enum import Enum, auto
  12 from hashlib import pbkdf2_hmac
  13
  14 from .aes import aes_cbc_decrypt_bytes, aes_gcm_decrypt_and_verify_bytes
  15 from .compat import (
  16     compat_b64decode,
  17     compat_cookiejar_Cookie,
  18 )
  19 from .utils import (
  20     expand_path,
  21     Popen,
  22     YoutubeDLCookieJar,
  23 )
  24
  25 try:
  26     import sqlite3
  27     SQLITE_AVAILABLE = True
  28 except ImportError:
  29     # although sqlite3 is part of the standard library, it is possible to compile python without
  30     # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544
  31     SQLITE_AVAILABLE = False
  32
  33
  34 try:
  35     import secretstorage
  36     SECRETSTORAGE_AVAILABLE = True
  37 except ImportError:
  38     SECRETSTORAGE_AVAILABLE = False
  39     SECRETSTORAGE_UNAVAILABLE_REASON = (
  40         'as the `secretstorage` module is not installed. '
  41         'Please install by running `python3 -m pip install secretstorage`.')
  42 except Exception as _err:
  43     SECRETSTORAGE_AVAILABLE = False
  44     SECRETSTORAGE_UNAVAILABLE_REASON = f'as the `secretstorage` module could not be initialized. {_err}'
  45
  46
  47 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
  48 SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
  49
  50
  51 class YDLLogger:
  52     def __init__(self, ydl=None):
  53         self._ydl = ydl
  54
  55     def debug(self, message):
  56         if self._ydl:
  57             self._ydl.write_debug(message)
  58
  59     def info(self, message):
  60         if self._ydl:
  61             self._ydl.to_screen(f'[Cookies] {message}')
  62
  63     def warning(self, message, only_once=False):
  64         if self._ydl:
  65             self._ydl.report_warning(message, only_once)
  66
  67     def error(self, message):
  68         if self._ydl:
  69             self._ydl.report_error(message)
  70
  71
  72 def load_cookies(cookie_file, browser_specification, ydl):
  73     cookie_jars = []
  74     if browser_specification is not None:
  75         browser_name, profile, keyring = _parse_browser_specification(*browser_specification)
  76         cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring))
  77
  78     if cookie_file is not None:
  79         cookie_file = expand_path(cookie_file)
  80         jar = YoutubeDLCookieJar(cookie_file)
  81         if os.access(cookie_file, os.R_OK):
  82             jar.load(ignore_discard=True, ignore_expires=True)
  83         cookie_jars.append(jar)
  84
  85     return _merge_cookie_jars(cookie_jars)
  86
  87
  88 def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None):
  89     if browser_name == 'firefox':
  90         return _extract_firefox_cookies(profile, logger)
  91     elif browser_name == 'safari':
  92         return _extract_safari_cookies(profile, logger)
  93     elif browser_name in CHROMIUM_BASED_BROWSERS:
  94         return _extract_chrome_cookies(browser_name, profile, keyring, logger)
  95     else:
  96         raise ValueError('unknown browser: {}'.format(browser_name))
  97
  98
  99 def _extract_firefox_cookies(profile, logger):
 100     logger.info('Extracting cookies from firefox')
 101     if not SQLITE_AVAILABLE:
 102         logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
 103                        'Please use a python interpreter compiled with sqlite3 support')
 104         return YoutubeDLCookieJar()
 105
 106     if profile is None:
 107         search_root = _firefox_browser_dir()
 108     elif _is_path(profile):
 109         search_root = profile
 110     else:
 111         search_root = os.path.join(_firefox_browser_dir(), profile)
 112
 113     cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite')
 114     if cookie_database_path is None:
 115         raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
 116     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 117
 118     with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
 119         cursor = None
 120         try:
 121             cursor = _open_database_copy(cookie_database_path, tmpdir)
 122             cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
 123             jar = YoutubeDLCookieJar()
 124             for host, name, value, path, expiry, is_secure in cursor.fetchall():
 125                 cookie = compat_cookiejar_Cookie(
 126                     version=0, name=name, value=value, port=None, port_specified=False,
 127                     domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
 128                     path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
 129                     comment=None, comment_url=None, rest={})
 130                 jar.set_cookie(cookie)
 131             logger.info('Extracted {} cookies from firefox'.format(len(jar)))
 132             return jar
 133         finally:
 134             if cursor is not None:
 135                 cursor.connection.close()
 136
 137
 138 def _firefox_browser_dir():
 139     if sys.platform in ('linux', 'linux2'):
 140         return os.path.expanduser('~/.mozilla/firefox')
 141     elif sys.platform == 'win32':
 142         return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox\Profiles')
 143     elif sys.platform == 'darwin':
 144         return os.path.expanduser('~/Library/Application Support/Firefox')
 145     else:
 146         raise ValueError('unsupported platform: {}'.format(sys.platform))
 147
 148
 149 def _get_chromium_based_browser_settings(browser_name):
 150     # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md
 151     if sys.platform in ('linux', 'linux2'):
 152         config = _config_home()
 153         browser_dir = {
 154             'brave': os.path.join(config, 'BraveSoftware/Brave-Browser'),
 155             'chrome': os.path.join(config, 'google-chrome'),
 156             'chromium': os.path.join(config, 'chromium'),
 157             'edge': os.path.join(config, 'microsoft-edge'),
 158             'opera': os.path.join(config, 'opera'),
 159             'vivaldi': os.path.join(config, 'vivaldi'),
 160         }[browser_name]
 161
 162     elif sys.platform == 'win32':
 163         appdata_local = os.path.expandvars('%LOCALAPPDATA%')
 164         appdata_roaming = os.path.expandvars('%APPDATA%')
 165         browser_dir = {
 166             'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'),
 167             'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'),
 168             'chromium': os.path.join(appdata_local, r'Chromium\User Data'),
 169             'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'),
 170             'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'),
 171             'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'),
 172         }[browser_name]
 173
 174     elif sys.platform == 'darwin':
 175         appdata = os.path.expanduser('~/Library/Application Support')
 176         browser_dir = {
 177             'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'),
 178             'chrome': os.path.join(appdata, 'Google/Chrome'),
 179             'chromium': os.path.join(appdata, 'Chromium'),
 180             'edge': os.path.join(appdata, 'Microsoft Edge'),
 181             'opera': os.path.join(appdata, 'com.operasoftware.Opera'),
 182             'vivaldi': os.path.join(appdata, 'Vivaldi'),
 183         }[browser_name]
 184
 185     else:
 186         raise ValueError('unsupported platform: {}'.format(sys.platform))
 187
 188     # Linux keyring names can be determined by snooping on dbus while opening the browser in KDE:
 189     # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
 190     keyring_name = {
 191         'brave': 'Brave',
 192         'chrome': 'Chrome',
 193         'chromium': 'Chromium',
 194         'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium',
 195         'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium',
 196         'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome',
 197     }[browser_name]
 198
 199     browsers_without_profiles = {'opera'}
 200
 201     return {
 202         'browser_dir': browser_dir,
 203         'keyring_name': keyring_name,
 204         'supports_profiles': browser_name not in browsers_without_profiles
 205     }
 206
 207
 208 def _extract_chrome_cookies(browser_name, profile, keyring, logger):
 209     logger.info('Extracting cookies from {}'.format(browser_name))
 210
 211     if not SQLITE_AVAILABLE:
 212         logger.warning(('Cannot extract cookies from {} without sqlite3 support. '
 213                         'Please use a python interpreter compiled with sqlite3 support').format(browser_name))
 214         return YoutubeDLCookieJar()
 215
 216     config = _get_chromium_based_browser_settings(browser_name)
 217
 218     if profile is None:
 219         search_root = config['browser_dir']
 220     elif _is_path(profile):
 221         search_root = profile
 222         config['browser_dir'] = os.path.dirname(profile) if config['supports_profiles'] else profile
 223     else:
 224         if config['supports_profiles']:
 225             search_root = os.path.join(config['browser_dir'], profile)
 226         else:
 227             logger.error('{} does not support profiles'.format(browser_name))
 228             search_root = config['browser_dir']
 229
 230     cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies')
 231     if cookie_database_path is None:
 232         raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
 233     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 234
 235     decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger, keyring=keyring)
 236
 237     with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
 238         cursor = None
 239         try:
 240             cursor = _open_database_copy(cookie_database_path, tmpdir)
 241             cursor.connection.text_factory = bytes
 242             column_names = _get_column_names(cursor, 'cookies')
 243             secure_column = 'is_secure' if 'is_secure' in column_names else 'secure'
 244             cursor.execute('SELECT host_key, name, value, encrypted_value, path, '
 245                            'expires_utc, {} FROM cookies'.format(secure_column))
 246             jar = YoutubeDLCookieJar()
 247             failed_cookies = 0
 248             unencrypted_cookies = 0
 249             for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
 250                 host_key = host_key.decode('utf-8')
 251                 name = name.decode('utf-8')
 252                 value = value.decode('utf-8')
 253                 path = path.decode('utf-8')
 254
 255                 if not value and encrypted_value:
 256                     value = decryptor.decrypt(encrypted_value)
 257                     if value is None:
 258                         failed_cookies += 1
 259                         continue
 260                 else:
 261                     unencrypted_cookies += 1
 262
 263                 cookie = compat_cookiejar_Cookie(
 264                     version=0, name=name, value=value, port=None, port_specified=False,
 265                     domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
 266                     path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
 267                     comment=None, comment_url=None, rest={})
 268                 jar.set_cookie(cookie)
 269             if failed_cookies > 0:
 270                 failed_message = ' ({} could not be decrypted)'.format(failed_cookies)
 271             else:
 272                 failed_message = ''
 273             logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message))
 274             counts = decryptor.cookie_counts.copy()
 275             counts['unencrypted'] = unencrypted_cookies
 276             logger.debug('cookie version breakdown: {}'.format(counts))
 277             return jar
 278         finally:
 279             if cursor is not None:
 280                 cursor.connection.close()
 281
 282
 283 class ChromeCookieDecryptor:
 284     """
 285     Overview:
 286
 287         Linux:
 288         - cookies are either v10 or v11
 289             - v10: AES-CBC encrypted with a fixed key
 290             - v11: AES-CBC encrypted with an OS protected key (keyring)
 291             - v11 keys can be stored in various places depending on the activate desktop environment [2]
 292
 293         Mac:
 294         - cookies are either v10 or not v10
 295             - v10: AES-CBC encrypted with an OS protected key (keyring) and more key derivation iterations than linux
 296             - not v10: 'old data' stored as plaintext
 297
 298         Windows:
 299         - cookies are either v10 or not v10
 300             - v10: AES-GCM encrypted with a key which is encrypted with DPAPI
 301             - not v10: encrypted with DPAPI
 302
 303     Sources:
 304     - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
 305     - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc
 306         - KeyStorageLinux::CreateService
 307     """
 308
 309     def decrypt(self, encrypted_value):
 310         raise NotImplementedError
 311
 312     @property
 313     def cookie_counts(self):
 314         raise NotImplementedError
 315
 316
 317 def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None):
 318     if sys.platform in ('linux', 'linux2'):
 319         return LinuxChromeCookieDecryptor(browser_keyring_name, logger, keyring=keyring)
 320     elif sys.platform == 'darwin':
 321         return MacChromeCookieDecryptor(browser_keyring_name, logger)
 322     elif sys.platform == 'win32':
 323         return WindowsChromeCookieDecryptor(browser_root, logger)
 324     else:
 325         raise NotImplementedError('Chrome cookie decryption is not supported '
 326                                   'on this platform: {}'.format(sys.platform))
 327
 328
 329 class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
 330     def __init__(self, browser_keyring_name, logger, *, keyring=None):
 331         self._logger = logger
 332         self._v10_key = self.derive_key(b'peanuts')
 333         password = _get_linux_keyring_password(browser_keyring_name, keyring, logger)
 334         self._v11_key = None if password is None else self.derive_key(password)
 335         self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
 336
 337     @staticmethod
 338     def derive_key(password):
 339         # values from
 340         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
 341         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
 342
 343     @property
 344     def cookie_counts(self):
 345         return self._cookie_counts
 346
 347     def decrypt(self, encrypted_value):
 348         version = encrypted_value[:3]
 349         ciphertext = encrypted_value[3:]
 350
 351         if version == b'v10':
 352             self._cookie_counts['v10'] += 1
 353             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 354
 355         elif version == b'v11':
 356             self._cookie_counts['v11'] += 1
 357             if self._v11_key is None:
 358                 self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True)
 359                 return None
 360             return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
 361
 362         else:
 363             self._cookie_counts['other'] += 1
 364             return None
 365
 366
 367 class MacChromeCookieDecryptor(ChromeCookieDecryptor):
 368     def __init__(self, browser_keyring_name, logger):
 369         self._logger = logger
 370         password = _get_mac_keyring_password(browser_keyring_name, logger)
 371         self._v10_key = None if password is None else self.derive_key(password)
 372         self._cookie_counts = {'v10': 0, 'other': 0}
 373
 374     @staticmethod
 375     def derive_key(password):
 376         # values from
 377         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
 378         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
 379
 380     @property
 381     def cookie_counts(self):
 382         return self._cookie_counts
 383
 384     def decrypt(self, encrypted_value):
 385         version = encrypted_value[:3]
 386         ciphertext = encrypted_value[3:]
 387
 388         if version == b'v10':
 389             self._cookie_counts['v10'] += 1
 390             if self._v10_key is None:
 391                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
 392                 return None
 393
 394             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 395
 396         else:
 397             self._cookie_counts['other'] += 1
 398             # other prefixes are considered 'old data' which were stored as plaintext
 399             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
 400             return encrypted_value
 401
 402
 403 class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
 404     def __init__(self, browser_root, logger):
 405         self._logger = logger
 406         self._v10_key = _get_windows_v10_key(browser_root, logger)
 407         self._cookie_counts = {'v10': 0, 'other': 0}
 408
 409     @property
 410     def cookie_counts(self):
 411         return self._cookie_counts
 412
 413     def decrypt(self, encrypted_value):
 414         version = encrypted_value[:3]
 415         ciphertext = encrypted_value[3:]
 416
 417         if version == b'v10':
 418             self._cookie_counts['v10'] += 1
 419             if self._v10_key is None:
 420                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
 421                 return None
 422
 423             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
 424             #   kNonceLength
 425             nonce_length = 96 // 8
 426             # boringssl
 427             #   EVP_AEAD_AES_GCM_TAG_LEN
 428             authentication_tag_length = 16
 429
 430             raw_ciphertext = ciphertext
 431             nonce = raw_ciphertext[:nonce_length]
 432             ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length]
 433             authentication_tag = raw_ciphertext[-authentication_tag_length:]
 434
 435             return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
 436
 437         else:
 438             self._cookie_counts['other'] += 1
 439             # any other prefix means the data is DPAPI encrypted
 440             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
 441             return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8')
 442
 443
 444 def _extract_safari_cookies(profile, logger):
 445     if profile is not None:
 446         logger.error('safari does not support profiles')
 447     if sys.platform != 'darwin':
 448         raise ValueError('unsupported platform: {}'.format(sys.platform))
 449
 450     cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
 451
 452     if not os.path.isfile(cookies_path):
 453         raise FileNotFoundError('could not find safari cookies database')
 454
 455     with open(cookies_path, 'rb') as f:
 456         cookies_data = f.read()
 457
 458     jar = parse_safari_cookies(cookies_data, logger=logger)
 459     logger.info('Extracted {} cookies from safari'.format(len(jar)))
 460     return jar
 461
 462
 463 class ParserError(Exception):
 464     pass
 465
 466
 467 class DataParser:
 468     def __init__(self, data, logger):
 469         self._data = data
 470         self.cursor = 0
 471         self._logger = logger
 472
 473     def read_bytes(self, num_bytes):
 474         if num_bytes < 0:
 475             raise ParserError('invalid read of {} bytes'.format(num_bytes))
 476         end = self.cursor + num_bytes
 477         if end > len(self._data):
 478             raise ParserError('reached end of input')
 479         data = self._data[self.cursor:end]
 480         self.cursor = end
 481         return data
 482
 483     def expect_bytes(self, expected_value, message):
 484         value = self.read_bytes(len(expected_value))
 485         if value != expected_value:
 486             raise ParserError('unexpected value: {} != {} ({})'.format(value, expected_value, message))
 487
 488     def read_uint(self, big_endian=False):
 489         data_format = '>I' if big_endian else '<I'
 490         return struct.unpack(data_format, self.read_bytes(4))[0]
 491
 492     def read_double(self, big_endian=False):
 493         data_format = '>d' if big_endian else '<d'
 494         return struct.unpack(data_format, self.read_bytes(8))[0]
 495
 496     def read_cstring(self):
 497         buffer = []
 498         while True:
 499             c = self.read_bytes(1)
 500             if c == b'\x00':
 501                 return b''.join(buffer).decode('utf-8')
 502             else:
 503                 buffer.append(c)
 504
 505     def skip(self, num_bytes, description='unknown'):
 506         if num_bytes > 0:
 507             self._logger.debug('skipping {} bytes ({}): {}'.format(
 508                 num_bytes, description, self.read_bytes(num_bytes)))
 509         elif num_bytes < 0:
 510             raise ParserError('invalid skip of {} bytes'.format(num_bytes))
 511
 512     def skip_to(self, offset, description='unknown'):
 513         self.skip(offset - self.cursor, description)
 514
 515     def skip_to_end(self, description='unknown'):
 516         self.skip_to(len(self._data), description)
 517
 518
 519 def _mac_absolute_time_to_posix(timestamp):
 520     return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp())
 521
 522
 523 def _parse_safari_cookies_header(data, logger):
 524     p = DataParser(data, logger)
 525     p.expect_bytes(b'cook', 'database signature')
 526     number_of_pages = p.read_uint(big_endian=True)
 527     page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)]
 528     return page_sizes, p.cursor
 529
 530
 531 def _parse_safari_cookies_page(data, jar, logger):
 532     p = DataParser(data, logger)
 533     p.expect_bytes(b'\x00\x00\x01\x00', 'page signature')
 534     number_of_cookies = p.read_uint()
 535     record_offsets = [p.read_uint() for _ in range(number_of_cookies)]
 536     if number_of_cookies == 0:
 537         logger.debug('a cookies page of size {} has no cookies'.format(len(data)))
 538         return
 539
 540     p.skip_to(record_offsets[0], 'unknown page header field')
 541
 542     for record_offset in record_offsets:
 543         p.skip_to(record_offset, 'space between records')
 544         record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
 545         p.read_bytes(record_length)
 546     p.skip_to_end('space in between pages')
 547
 548
 549 def _parse_safari_cookies_record(data, jar, logger):
 550     p = DataParser(data, logger)
 551     record_size = p.read_uint()
 552     p.skip(4, 'unknown record field 1')
 553     flags = p.read_uint()
 554     is_secure = bool(flags & 0x0001)
 555     p.skip(4, 'unknown record field 2')
 556     domain_offset = p.read_uint()
 557     name_offset = p.read_uint()
 558     path_offset = p.read_uint()
 559     value_offset = p.read_uint()
 560     p.skip(8, 'unknown record field 3')
 561     expiration_date = _mac_absolute_time_to_posix(p.read_double())
 562     _creation_date = _mac_absolute_time_to_posix(p.read_double())  # noqa: F841
 563
 564     try:
 565         p.skip_to(domain_offset)
 566         domain = p.read_cstring()
 567
 568         p.skip_to(name_offset)
 569         name = p.read_cstring()
 570
 571         p.skip_to(path_offset)
 572         path = p.read_cstring()
 573
 574         p.skip_to(value_offset)
 575         value = p.read_cstring()
 576     except UnicodeDecodeError:
 577         logger.warning('failed to parse Safari cookie because UTF-8 decoding failed', only_once=True)
 578         return record_size
 579
 580     p.skip_to(record_size, 'space at the end of the record')
 581
 582     cookie = compat_cookiejar_Cookie(
 583         version=0, name=name, value=value, port=None, port_specified=False,
 584         domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'),
 585         path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False,
 586         comment=None, comment_url=None, rest={})
 587     jar.set_cookie(cookie)
 588     return record_size
 589
 590
 591 def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
 592     """
 593     References:
 594         - https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc
 595             - this data appears to be out of date but the important parts of the database structure is the same
 596             - there are a few bytes here and there which are skipped during parsing
 597     """
 598     if jar is None:
 599         jar = YoutubeDLCookieJar()
 600     page_sizes, body_start = _parse_safari_cookies_header(data, logger)
 601     p = DataParser(data[body_start:], logger)
 602     for page_size in page_sizes:
 603         _parse_safari_cookies_page(p.read_bytes(page_size), jar, logger)
 604     p.skip_to_end('footer')
 605     return jar
 606
 607
 608 class _LinuxDesktopEnvironment(Enum):
 609     """
 610     https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.h
 611     DesktopEnvironment
 612     """
 613     OTHER = auto()
 614     CINNAMON = auto()
 615     GNOME = auto()
 616     KDE = auto()
 617     PANTHEON = auto()
 618     UNITY = auto()
 619     XFCE = auto()
 620
 621
 622 class _LinuxKeyring(Enum):
 623     """
 624     https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.h
 625     SelectedLinuxBackend
 626     """
 627     KWALLET = auto()
 628     GNOMEKEYRING = auto()
 629     BASICTEXT = auto()
 630
 631
 632 SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
 633
 634
 635 def _get_linux_desktop_environment(env):
 636     """
 637     https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.cc
 638     GetDesktopEnvironment
 639     """
 640     xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None)
 641     desktop_session = env.get('DESKTOP_SESSION', None)
 642     if xdg_current_desktop is not None:
 643         xdg_current_desktop = xdg_current_desktop.split(':')[0].strip()
 644
 645         if xdg_current_desktop == 'Unity':
 646             if desktop_session is not None and 'gnome-fallback' in desktop_session:
 647                 return _LinuxDesktopEnvironment.GNOME
 648             else:
 649                 return _LinuxDesktopEnvironment.UNITY
 650         elif xdg_current_desktop == 'GNOME':
 651             return _LinuxDesktopEnvironment.GNOME
 652         elif xdg_current_desktop == 'X-Cinnamon':
 653             return _LinuxDesktopEnvironment.CINNAMON
 654         elif xdg_current_desktop == 'KDE':
 655             return _LinuxDesktopEnvironment.KDE
 656         elif xdg_current_desktop == 'Pantheon':
 657             return _LinuxDesktopEnvironment.PANTHEON
 658         elif xdg_current_desktop == 'XFCE':
 659             return _LinuxDesktopEnvironment.XFCE
 660     elif desktop_session is not None:
 661         if desktop_session in ('mate', 'gnome'):
 662             return _LinuxDesktopEnvironment.GNOME
 663         elif 'kde' in desktop_session:
 664             return _LinuxDesktopEnvironment.KDE
 665         elif 'xfce' in desktop_session:
 666             return _LinuxDesktopEnvironment.XFCE
 667     else:
 668         if 'GNOME_DESKTOP_SESSION_ID' in env:
 669             return _LinuxDesktopEnvironment.GNOME
 670         elif 'KDE_FULL_SESSION' in env:
 671             return _LinuxDesktopEnvironment.KDE
 672         else:
 673             return _LinuxDesktopEnvironment.OTHER
 674
 675
 676 def _choose_linux_keyring(logger):
 677     """
 678     https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.cc
 679     SelectBackend
 680     """
 681     desktop_environment = _get_linux_desktop_environment(os.environ)
 682     logger.debug('detected desktop environment: {}'.format(desktop_environment.name))
 683     if desktop_environment == _LinuxDesktopEnvironment.KDE:
 684         linux_keyring = _LinuxKeyring.KWALLET
 685     elif desktop_environment == _LinuxDesktopEnvironment.OTHER:
 686         linux_keyring = _LinuxKeyring.BASICTEXT
 687     else:
 688         linux_keyring = _LinuxKeyring.GNOMEKEYRING
 689     return linux_keyring
 690
 691
 692 def _get_kwallet_network_wallet(logger):
 693     """ The name of the wallet used to store network passwords.
 694
 695     https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/kwallet_dbus.cc
 696     KWalletDBus::NetworkWallet
 697     which does a dbus call to the following function:
 698     https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
 699     Wallet::NetworkWallet
 700     """
 701     default_wallet = 'kdewallet'
 702     try:
 703         proc = Popen([
 704             'dbus-send', '--session', '--print-reply=literal',
 705             '--dest=org.kde.kwalletd5',
 706             '/modules/kwalletd5',
 707             'org.kde.KWallet.networkWallet'
 708         ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
 709
 710         stdout, stderr = proc.communicate_or_kill()
 711         if proc.returncode != 0:
 712             logger.warning('failed to read NetworkWallet')
 713             return default_wallet
 714         else:
 715             network_wallet = stdout.decode('utf-8').strip()
 716             logger.debug('NetworkWallet = "{}"'.format(network_wallet))
 717             return network_wallet
 718     except BaseException as e:
 719         logger.warning('exception while obtaining NetworkWallet: {}'.format(e))
 720         return default_wallet
 721
 722
 723 def _get_kwallet_password(browser_keyring_name, logger):
 724     logger.debug('using kwallet-query to obtain password from kwallet')
 725
 726     if shutil.which('kwallet-query') is None:
 727         logger.error('kwallet-query command not found. KWallet and kwallet-query '
 728                      'must be installed to read from KWallet. kwallet-query should be'
 729                      'included in the kwallet package for your distribution')
 730         return b''
 731
 732     network_wallet = _get_kwallet_network_wallet(logger)
 733
 734     try:
 735         proc = Popen([
 736             'kwallet-query',
 737             '--read-password', '{} Safe Storage'.format(browser_keyring_name),
 738             '--folder', '{} Keys'.format(browser_keyring_name),
 739             network_wallet
 740         ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
 741
 742         stdout, stderr = proc.communicate_or_kill()
 743         if proc.returncode != 0:
 744             logger.error('kwallet-query failed with return code {}. Please consult '
 745                          'the kwallet-query man page for details'.format(proc.returncode))
 746             return b''
 747         else:
 748             if stdout.lower().startswith(b'failed to read'):
 749                 logger.debug('failed to read password from kwallet. Using empty string instead')
 750                 # this sometimes occurs in KDE because chrome does not check hasEntry and instead
 751                 # just tries to read the value (which kwallet returns "") whereas kwallet-query
 752                 # checks hasEntry. To verify this:
 753                 # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
 754                 # while starting chrome.
 755                 # this may be a bug as the intended behaviour is to generate a random password and store
 756                 # it, but that doesn't matter here.
 757                 return b''
 758             else:
 759                 logger.debug('password found')
 760                 if stdout[-1:] == b'\n':
 761                     stdout = stdout[:-1]
 762                 return stdout
 763     except BaseException as e:
 764         logger.warning(f'exception running kwallet-query: {type(e).__name__}({e})')
 765         return b''
 766
 767
 768 def _get_gnome_keyring_password(browser_keyring_name, logger):
 769     if not SECRETSTORAGE_AVAILABLE:
 770         logger.error('secretstorage not available {}'.format(SECRETSTORAGE_UNAVAILABLE_REASON))
 771         return b''
 772     # the Gnome keyring does not seem to organise keys in the same way as KWallet,
 773     # using `dbus-monitor` during startup, it can be observed that chromium lists all keys
 774     # and presumably searches for its key in the list. It appears that we must do the same.
 775     # https://github.com/jaraco/keyring/issues/556
 776     with contextlib.closing(secretstorage.dbus_init()) as con:
 777         col = secretstorage.get_default_collection(con)
 778         for item in col.get_all_items():
 779             if item.get_label() == '{} Safe Storage'.format(browser_keyring_name):
 780                 return item.get_secret()
 781         else:
 782             logger.error('failed to read from keyring')
 783             return b''
 784
 785
 786 def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
 787     # note: chrome/chromium can be run with the following flags to determine which keyring backend
 788     # it has chosen to use
 789     # chromium --enable-logging=stderr --v=1 2>&1 | grep key_storage_
 790     # Chromium supports a flag: --password-store=<basic|gnome|kwallet> so the automatic detection
 791     # will not be sufficient in all cases.
 792
 793     keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
 794     logger.debug(f'Chosen keyring: {keyring.name}')
 795
 796     if keyring == _LinuxKeyring.KWALLET:
 797         return _get_kwallet_password(browser_keyring_name, logger)
 798     elif keyring == _LinuxKeyring.GNOMEKEYRING:
 799         return _get_gnome_keyring_password(browser_keyring_name, logger)
 800     elif keyring == _LinuxKeyring.BASICTEXT:
 801         # when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
 802         return None
 803     assert False, f'Unknown keyring {keyring}'
 804
 805
 806 def _get_mac_keyring_password(browser_keyring_name, logger):
 807     logger.debug('using find-generic-password to obtain password from OSX keychain')
 808     try:
 809         proc = Popen(
 810             ['security', 'find-generic-password',
 811              '-w',  # write password to stdout
 812              '-a', browser_keyring_name,  # match 'account'
 813              '-s', '{} Safe Storage'.format(browser_keyring_name)],  # match 'service'
 814             stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
 815
 816         stdout, stderr = proc.communicate_or_kill()
 817         if stdout[-1:] == b'\n':
 818             stdout = stdout[:-1]
 819         return stdout
 820     except BaseException as e:
 821         logger.warning(f'exception running find-generic-password: {type(e).__name__}({e})')
 822         return None
 823
 824
 825 def _get_windows_v10_key(browser_root, logger):
 826     path = _find_most_recently_used_file(browser_root, 'Local State')
 827     if path is None:
 828         logger.error('could not find local state file')
 829         return None
 830     with open(path, 'r', encoding='utf8') as f:
 831         data = json.load(f)
 832     try:
 833         base64_key = data['os_crypt']['encrypted_key']
 834     except KeyError:
 835         logger.error('no encrypted key in Local State')
 836         return None
 837     encrypted_key = compat_b64decode(base64_key)
 838     prefix = b'DPAPI'
 839     if not encrypted_key.startswith(prefix):
 840         logger.error('invalid key')
 841         return None
 842     return _decrypt_windows_dpapi(encrypted_key[len(prefix):], logger)
 843
 844
 845 def pbkdf2_sha1(password, salt, iterations, key_length):
 846     return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
 847
 848
 849 def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
 850     plaintext = aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector)
 851     padding_length = plaintext[-1]
 852     try:
 853         return plaintext[:-padding_length].decode('utf-8')
 854     except UnicodeDecodeError:
 855         logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
 856         return None
 857
 858
 859 def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
 860     try:
 861         plaintext = aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce)
 862     except ValueError:
 863         logger.warning('failed to decrypt cookie (AES-GCM) because the MAC check failed. Possibly the key is wrong?', only_once=True)
 864         return None
 865
 866     try:
 867         return plaintext.decode('utf-8')
 868     except UnicodeDecodeError:
 869         logger.warning('failed to decrypt cookie (AES-GCM) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
 870         return None
 871
 872
 873 def _decrypt_windows_dpapi(ciphertext, logger):
 874     """
 875     References:
 876         - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata
 877     """
 878     from ctypes.wintypes import DWORD
 879
 880     class DATA_BLOB(ctypes.Structure):
 881         _fields_ = [('cbData', DWORD),
 882                     ('pbData', ctypes.POINTER(ctypes.c_char))]
 883
 884     buffer = ctypes.create_string_buffer(ciphertext)
 885     blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer)
 886     blob_out = DATA_BLOB()
 887     ret = ctypes.windll.crypt32.CryptUnprotectData(
 888         ctypes.byref(blob_in),  # pDataIn
 889         None,  # ppszDataDescr: human readable description of pDataIn
 890         None,  # pOptionalEntropy: salt?
 891         None,  # pvReserved: must be NULL
 892         None,  # pPromptStruct: information about prompts to display
 893         0,  # dwFlags
 894         ctypes.byref(blob_out)  # pDataOut
 895     )
 896     if not ret:
 897         logger.warning('failed to decrypt with DPAPI', only_once=True)
 898         return None
 899
 900     result = ctypes.string_at(blob_out.pbData, blob_out.cbData)
 901     ctypes.windll.kernel32.LocalFree(blob_out.pbData)
 902     return result
 903
 904
 905 def _config_home():
 906     return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
 907
 908
 909 def _open_database_copy(database_path, tmpdir):
 910     # cannot open sqlite databases if they are already in use (e.g. by the browser)
 911     database_copy_path = os.path.join(tmpdir, 'temporary.sqlite')
 912     shutil.copy(database_path, database_copy_path)
 913     conn = sqlite3.connect(database_copy_path)
 914     return conn.cursor()
 915
 916
 917 def _get_column_names(cursor, table_name):
 918     table_info = cursor.execute('PRAGMA table_info({})'.format(table_name)).fetchall()
 919     return [row[1].decode('utf-8') for row in table_info]
 920
 921
 922 def _find_most_recently_used_file(root, filename):
 923     # if there are multiple browser profiles, take the most recently used one
 924     paths = []
 925     for root, dirs, files in os.walk(root):
 926         for file in files:
 927             if file == filename:
 928                 paths.append(os.path.join(root, file))
 929     return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
 930
 931
 932 def _merge_cookie_jars(jars):
 933     output_jar = YoutubeDLCookieJar()
 934     for jar in jars:
 935         for cookie in jar:
 936             output_jar.set_cookie(cookie)
 937         if jar.filename is not None:
 938             output_jar.filename = jar.filename
 939     return output_jar
 940
 941
 942 def _is_path(value):
 943     return os.path.sep in value
 944
 945
 946 def _parse_browser_specification(browser_name, profile=None, keyring=None):
 947     if browser_name not in SUPPORTED_BROWSERS:
 948         raise ValueError(f'unsupported browser: "{browser_name}"')
 949     if keyring not in (None, *SUPPORTED_KEYRINGS):
 950         raise ValueError(f'unsupported keyring: "{keyring}"')
 951     if profile is not None and _is_path(profile):
 952         profile = os.path.expanduser(profile)
 953     return browser_name, profile, keyring