yt_dlp/cookies.py

   1 import ctypes
   2 import json
   3 import os
   4 import shutil
   5 import struct
   6 import subprocess
   7 import sys
   8 import tempfile
   9 from datetime import datetime, timedelta, timezone
  10 from hashlib import pbkdf2_hmac
  11
  12 from yt_dlp.aes import aes_cbc_decrypt
  13 from yt_dlp.compat import (
  14     compat_b64decode,
  15     compat_cookiejar_Cookie,
  16 )
  17 from yt_dlp.utils import (
  18     bug_reports_message,
  19     bytes_to_intlist,
  20     expand_path,
  21     intlist_to_bytes,
  22     process_communicate_or_kill,
  23     YoutubeDLCookieJar,
  24 )
  25
  26 try:
  27     import sqlite3
  28     SQLITE_AVAILABLE = True
  29 except ImportError:
  30     # although sqlite3 is part of the standard library, it is possible to compile python without
  31     # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544
  32     SQLITE_AVAILABLE = False
  33
  34
  35 try:
  36     from Crypto.Cipher import AES
  37     CRYPTO_AVAILABLE = True
  38 except ImportError:
  39     CRYPTO_AVAILABLE = False
  40
  41 try:
  42     import keyring
  43     KEYRING_AVAILABLE = True
  44     KEYRING_UNAVAILABLE_REASON = f'due to unknown reasons{bug_reports_message()}'
  45 except ImportError:
  46     KEYRING_AVAILABLE = False
  47     KEYRING_UNAVAILABLE_REASON = (
  48         'as the `keyring` module is not installed. '
  49         'Please install by running `python3 -m pip install keyring`. '
  50         'Depending on your platform, additional packages may be required '
  51         'to access the keyring; see  https://pypi.org/project/keyring')
  52 except Exception as _err:
  53     KEYRING_AVAILABLE = False
  54     KEYRING_UNAVAILABLE_REASON = 'as the `keyring` module could not be initialized: %s' % _err
  55
  56
  57 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
  58 SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
  59
  60
  61 class YDLLogger:
  62     def __init__(self, ydl=None):
  63         self._ydl = ydl
  64
  65     def debug(self, message):
  66         if self._ydl:
  67             self._ydl.write_debug(message)
  68
  69     def info(self, message):
  70         if self._ydl:
  71             self._ydl.to_screen(f'[Cookies] {message}')
  72
  73     def warning(self, message, only_once=False):
  74         if self._ydl:
  75             self._ydl.report_warning(message, only_once)
  76
  77     def error(self, message):
  78         if self._ydl:
  79             self._ydl.report_error(message)
  80
  81
  82 def load_cookies(cookie_file, browser_specification, ydl):
  83     cookie_jars = []
  84     if browser_specification is not None:
  85         browser_name, profile = _parse_browser_specification(*browser_specification)
  86         cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl)))
  87
  88     if cookie_file is not None:
  89         cookie_file = expand_path(cookie_file)
  90         jar = YoutubeDLCookieJar(cookie_file)
  91         if os.access(cookie_file, os.R_OK):
  92             jar.load(ignore_discard=True, ignore_expires=True)
  93         cookie_jars.append(jar)
  94
  95     return _merge_cookie_jars(cookie_jars)
  96
  97
  98 def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger()):
  99     if browser_name == 'firefox':
 100         return _extract_firefox_cookies(profile, logger)
 101     elif browser_name == 'safari':
 102         return _extract_safari_cookies(profile, logger)
 103     elif browser_name in CHROMIUM_BASED_BROWSERS:
 104         return _extract_chrome_cookies(browser_name, profile, logger)
 105     else:
 106         raise ValueError('unknown browser: {}'.format(browser_name))
 107
 108
 109 def _extract_firefox_cookies(profile, logger):
 110     logger.info('Extracting cookies from firefox')
 111     if not SQLITE_AVAILABLE:
 112         logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
 113                        'Please use a python interpreter compiled with sqlite3 support')
 114         return YoutubeDLCookieJar()
 115
 116     if profile is None:
 117         search_root = _firefox_browser_dir()
 118     elif _is_path(profile):
 119         search_root = profile
 120     else:
 121         search_root = os.path.join(_firefox_browser_dir(), profile)
 122
 123     cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite')
 124     if cookie_database_path is None:
 125         raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
 126     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 127
 128     with tempfile.TemporaryDirectory(prefix='youtube_dl') as tmpdir:
 129         cursor = None
 130         try:
 131             cursor = _open_database_copy(cookie_database_path, tmpdir)
 132             cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
 133             jar = YoutubeDLCookieJar()
 134             for host, name, value, path, expiry, is_secure in cursor.fetchall():
 135                 cookie = compat_cookiejar_Cookie(
 136                     version=0, name=name, value=value, port=None, port_specified=False,
 137                     domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
 138                     path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
 139                     comment=None, comment_url=None, rest={})
 140                 jar.set_cookie(cookie)
 141             logger.info('Extracted {} cookies from firefox'.format(len(jar)))
 142             return jar
 143         finally:
 144             if cursor is not None:
 145                 cursor.connection.close()
 146
 147
 148 def _firefox_browser_dir():
 149     if sys.platform in ('linux', 'linux2'):
 150         return os.path.expanduser('~/.mozilla/firefox')
 151     elif sys.platform == 'win32':
 152         return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox\Profiles')
 153     elif sys.platform == 'darwin':
 154         return os.path.expanduser('~/Library/Application Support/Firefox')
 155     else:
 156         raise ValueError('unsupported platform: {}'.format(sys.platform))
 157
 158
 159 def _get_chromium_based_browser_settings(browser_name):
 160     # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md
 161     if sys.platform in ('linux', 'linux2'):
 162         config = _config_home()
 163         browser_dir = {
 164             'brave': os.path.join(config, 'BraveSoftware/Brave-Browser'),
 165             'chrome': os.path.join(config, 'google-chrome'),
 166             'chromium': os.path.join(config, 'chromium'),
 167             'edge': os.path.join(config, 'microsoft-edge'),
 168             'opera': os.path.join(config, 'opera'),
 169             'vivaldi': os.path.join(config, 'vivaldi'),
 170         }[browser_name]
 171
 172     elif sys.platform == 'win32':
 173         appdata_local = os.path.expandvars('%LOCALAPPDATA%')
 174         appdata_roaming = os.path.expandvars('%APPDATA%')
 175         browser_dir = {
 176             'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'),
 177             'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'),
 178             'chromium': os.path.join(appdata_local, r'Chromium\User Data'),
 179             'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'),
 180             'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'),
 181             'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'),
 182         }[browser_name]
 183
 184     elif sys.platform == 'darwin':
 185         appdata = os.path.expanduser('~/Library/Application Support')
 186         browser_dir = {
 187             'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'),
 188             'chrome': os.path.join(appdata, 'Google/Chrome'),
 189             'chromium': os.path.join(appdata, 'Chromium'),
 190             'edge': os.path.join(appdata, 'Microsoft Edge'),
 191             'opera': os.path.join(appdata, 'com.operasoftware.Opera'),
 192             'vivaldi': os.path.join(appdata, 'Vivaldi'),
 193         }[browser_name]
 194
 195     else:
 196         raise ValueError('unsupported platform: {}'.format(sys.platform))
 197
 198     # Linux keyring names can be determined by snooping on dbus while opening the browser in KDE:
 199     # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
 200     keyring_name = {
 201         'brave': 'Brave',
 202         'chrome': 'Chrome',
 203         'chromium': 'Chromium',
 204         'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium',
 205         'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium',
 206         'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome',
 207     }[browser_name]
 208
 209     browsers_without_profiles = {'opera'}
 210
 211     return {
 212         'browser_dir': browser_dir,
 213         'keyring_name': keyring_name,
 214         'supports_profiles': browser_name not in browsers_without_profiles
 215     }
 216
 217
 218 def _extract_chrome_cookies(browser_name, profile, logger):
 219     logger.info('Extracting cookies from {}'.format(browser_name))
 220
 221     if not SQLITE_AVAILABLE:
 222         logger.warning(('Cannot extract cookies from {} without sqlite3 support. '
 223                         'Please use a python interpreter compiled with sqlite3 support').format(browser_name))
 224         return YoutubeDLCookieJar()
 225
 226     config = _get_chromium_based_browser_settings(browser_name)
 227
 228     if profile is None:
 229         search_root = config['browser_dir']
 230     elif _is_path(profile):
 231         search_root = profile
 232         config['browser_dir'] = os.path.dirname(profile) if config['supports_profiles'] else profile
 233     else:
 234         if config['supports_profiles']:
 235             search_root = os.path.join(config['browser_dir'], profile)
 236         else:
 237             logger.error('{} does not support profiles'.format(browser_name))
 238             search_root = config['browser_dir']
 239
 240     cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies')
 241     if cookie_database_path is None:
 242         raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
 243     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 244
 245     decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger)
 246
 247     with tempfile.TemporaryDirectory(prefix='youtube_dl') as tmpdir:
 248         cursor = None
 249         try:
 250             cursor = _open_database_copy(cookie_database_path, tmpdir)
 251             cursor.connection.text_factory = bytes
 252             column_names = _get_column_names(cursor, 'cookies')
 253             secure_column = 'is_secure' if 'is_secure' in column_names else 'secure'
 254             cursor.execute('SELECT host_key, name, value, encrypted_value, path, '
 255                            'expires_utc, {} FROM cookies'.format(secure_column))
 256             jar = YoutubeDLCookieJar()
 257             failed_cookies = 0
 258             for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
 259                 host_key = host_key.decode('utf-8')
 260                 name = name.decode('utf-8')
 261                 value = value.decode('utf-8')
 262                 path = path.decode('utf-8')
 263
 264                 if not value and encrypted_value:
 265                     value = decryptor.decrypt(encrypted_value)
 266                     if value is None:
 267                         failed_cookies += 1
 268                         continue
 269
 270                 cookie = compat_cookiejar_Cookie(
 271                     version=0, name=name, value=value, port=None, port_specified=False,
 272                     domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
 273                     path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
 274                     comment=None, comment_url=None, rest={})
 275                 jar.set_cookie(cookie)
 276             if failed_cookies > 0:
 277                 failed_message = ' ({} could not be decrypted)'.format(failed_cookies)
 278             else:
 279                 failed_message = ''
 280             logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message))
 281             return jar
 282         finally:
 283             if cursor is not None:
 284                 cursor.connection.close()
 285
 286
 287 class ChromeCookieDecryptor:
 288     """
 289     Overview:
 290
 291         Linux:
 292         - cookies are either v10 or v11
 293             - v10: AES-CBC encrypted with a fixed key
 294             - v11: AES-CBC encrypted with an OS protected key (keyring)
 295             - v11 keys can be stored in various places depending on the activate desktop environment [2]
 296
 297         Mac:
 298         - cookies are either v10 or not v10
 299             - v10: AES-CBC encrypted with an OS protected key (keyring) and more key derivation iterations than linux
 300             - not v10: 'old data' stored as plaintext
 301
 302         Windows:
 303         - cookies are either v10 or not v10
 304             - v10: AES-GCM encrypted with a key which is encrypted with DPAPI
 305             - not v10: encrypted with DPAPI
 306
 307     Sources:
 308     - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
 309     - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc
 310         - KeyStorageLinux::CreateService
 311     """
 312
 313     def decrypt(self, encrypted_value):
 314         raise NotImplementedError
 315
 316
 317 def get_cookie_decryptor(browser_root, browser_keyring_name, logger):
 318     if sys.platform in ('linux', 'linux2'):
 319         return LinuxChromeCookieDecryptor(browser_keyring_name, logger)
 320     elif sys.platform == 'darwin':
 321         return MacChromeCookieDecryptor(browser_keyring_name, logger)
 322     elif sys.platform == 'win32':
 323         return WindowsChromeCookieDecryptor(browser_root, logger)
 324     else:
 325         raise NotImplementedError('Chrome cookie decryption is not supported '
 326                                   'on this platform: {}'.format(sys.platform))
 327
 328
 329 class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
 330     def __init__(self, browser_keyring_name, logger):
 331         self._logger = logger
 332         self._v10_key = self.derive_key(b'peanuts')
 333         if KEYRING_AVAILABLE:
 334             self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name))
 335         else:
 336             self._v11_key = None
 337
 338     @staticmethod
 339     def derive_key(password):
 340         # values from
 341         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
 342         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
 343
 344     def decrypt(self, encrypted_value):
 345         version = encrypted_value[:3]
 346         ciphertext = encrypted_value[3:]
 347
 348         if version == b'v10':
 349             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 350
 351         elif version == b'v11':
 352             if self._v11_key is None:
 353                 self._logger.warning(f'cannot decrypt cookie {KEYRING_UNAVAILABLE_REASON}', only_once=True)
 354                 return None
 355             return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
 356
 357         else:
 358             return None
 359
 360
 361 class MacChromeCookieDecryptor(ChromeCookieDecryptor):
 362     def __init__(self, browser_keyring_name, logger):
 363         self._logger = logger
 364         password = _get_mac_keyring_password(browser_keyring_name)
 365         self._v10_key = None if password is None else self.derive_key(password)
 366
 367     @staticmethod
 368     def derive_key(password):
 369         # values from
 370         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
 371         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
 372
 373     def decrypt(self, encrypted_value):
 374         version = encrypted_value[:3]
 375         ciphertext = encrypted_value[3:]
 376
 377         if version == b'v10':
 378             if self._v10_key is None:
 379                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
 380                 return None
 381
 382             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 383
 384         else:
 385             # other prefixes are considered 'old data' which were stored as plaintext
 386             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
 387             return encrypted_value
 388
 389
 390 class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
 391     def __init__(self, browser_root, logger):
 392         self._logger = logger
 393         self._v10_key = _get_windows_v10_key(browser_root, logger)
 394
 395     def decrypt(self, encrypted_value):
 396         version = encrypted_value[:3]
 397         ciphertext = encrypted_value[3:]
 398
 399         if version == b'v10':
 400             if self._v10_key is None:
 401                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
 402                 return None
 403             elif not CRYPTO_AVAILABLE:
 404                 self._logger.warning('cannot decrypt cookie as the `pycryptodome` module is not installed. '
 405                                      'Please install by running `python3 -m pip install pycryptodome`',
 406                                      only_once=True)
 407                 return None
 408
 409             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
 410             #   kNonceLength
 411             nonce_length = 96 // 8
 412             # boringssl
 413             #   EVP_AEAD_AES_GCM_TAG_LEN
 414             authentication_tag_length = 16
 415
 416             raw_ciphertext = ciphertext
 417             nonce = raw_ciphertext[:nonce_length]
 418             ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length]
 419             authentication_tag = raw_ciphertext[-authentication_tag_length:]
 420
 421             return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
 422
 423         else:
 424             # any other prefix means the data is DPAPI encrypted
 425             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
 426             return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8')
 427
 428
 429 def _extract_safari_cookies(profile, logger):
 430     if profile is not None:
 431         logger.error('safari does not support profiles')
 432     if sys.platform != 'darwin':
 433         raise ValueError('unsupported platform: {}'.format(sys.platform))
 434
 435     cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
 436
 437     if not os.path.isfile(cookies_path):
 438         raise FileNotFoundError('could not find safari cookies database')
 439
 440     with open(cookies_path, 'rb') as f:
 441         cookies_data = f.read()
 442
 443     jar = parse_safari_cookies(cookies_data, logger=logger)
 444     logger.info('Extracted {} cookies from safari'.format(len(jar)))
 445     return jar
 446
 447
 448 class ParserError(Exception):
 449     pass
 450
 451
 452 class DataParser:
 453     def __init__(self, data, logger):
 454         self._data = data
 455         self.cursor = 0
 456         self._logger = logger
 457
 458     def read_bytes(self, num_bytes):
 459         if num_bytes < 0:
 460             raise ParserError('invalid read of {} bytes'.format(num_bytes))
 461         end = self.cursor + num_bytes
 462         if end > len(self._data):
 463             raise ParserError('reached end of input')
 464         data = self._data[self.cursor:end]
 465         self.cursor = end
 466         return data
 467
 468     def expect_bytes(self, expected_value, message):
 469         value = self.read_bytes(len(expected_value))
 470         if value != expected_value:
 471             raise ParserError('unexpected value: {} != {} ({})'.format(value, expected_value, message))
 472
 473     def read_uint(self, big_endian=False):
 474         data_format = '>I' if big_endian else '<I'
 475         return struct.unpack(data_format, self.read_bytes(4))[0]
 476
 477     def read_double(self, big_endian=False):
 478         data_format = '>d' if big_endian else '<d'
 479         return struct.unpack(data_format, self.read_bytes(8))[0]
 480
 481     def read_cstring(self):
 482         buffer = []
 483         while True:
 484             c = self.read_bytes(1)
 485             if c == b'\x00':
 486                 return b''.join(buffer).decode('utf-8')
 487             else:
 488                 buffer.append(c)
 489
 490     def skip(self, num_bytes, description='unknown'):
 491         if num_bytes > 0:
 492             self._logger.debug('skipping {} bytes ({}): {}'.format(
 493                 num_bytes, description, self.read_bytes(num_bytes)))
 494         elif num_bytes < 0:
 495             raise ParserError('invalid skip of {} bytes'.format(num_bytes))
 496
 497     def skip_to(self, offset, description='unknown'):
 498         self.skip(offset - self.cursor, description)
 499
 500     def skip_to_end(self, description='unknown'):
 501         self.skip_to(len(self._data), description)
 502
 503
 504 def _mac_absolute_time_to_posix(timestamp):
 505     return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp())
 506
 507
 508 def _parse_safari_cookies_header(data, logger):
 509     p = DataParser(data, logger)
 510     p.expect_bytes(b'cook', 'database signature')
 511     number_of_pages = p.read_uint(big_endian=True)
 512     page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)]
 513     return page_sizes, p.cursor
 514
 515
 516 def _parse_safari_cookies_page(data, jar, logger):
 517     p = DataParser(data, logger)
 518     p.expect_bytes(b'\x00\x00\x01\x00', 'page signature')
 519     number_of_cookies = p.read_uint()
 520     record_offsets = [p.read_uint() for _ in range(number_of_cookies)]
 521     if number_of_cookies == 0:
 522         logger.debug('a cookies page of size {} has no cookies'.format(len(data)))
 523         return
 524
 525     p.skip_to(record_offsets[0], 'unknown page header field')
 526
 527     for record_offset in record_offsets:
 528         p.skip_to(record_offset, 'space between records')
 529         record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
 530         p.read_bytes(record_length)
 531     p.skip_to_end('space in between pages')
 532
 533
 534 def _parse_safari_cookies_record(data, jar, logger):
 535     p = DataParser(data, logger)
 536     record_size = p.read_uint()
 537     p.skip(4, 'unknown record field 1')
 538     flags = p.read_uint()
 539     is_secure = bool(flags & 0x0001)
 540     p.skip(4, 'unknown record field 2')
 541     domain_offset = p.read_uint()
 542     name_offset = p.read_uint()
 543     path_offset = p.read_uint()
 544     value_offset = p.read_uint()
 545     p.skip(8, 'unknown record field 3')
 546     expiration_date = _mac_absolute_time_to_posix(p.read_double())
 547     _creation_date = _mac_absolute_time_to_posix(p.read_double())  # noqa: F841
 548
 549     try:
 550         p.skip_to(domain_offset)
 551         domain = p.read_cstring()
 552
 553         p.skip_to(name_offset)
 554         name = p.read_cstring()
 555
 556         p.skip_to(path_offset)
 557         path = p.read_cstring()
 558
 559         p.skip_to(value_offset)
 560         value = p.read_cstring()
 561     except UnicodeDecodeError:
 562         logger.warning('failed to parse cookie because UTF-8 decoding failed', only_once=True)
 563         return record_size
 564
 565     p.skip_to(record_size, 'space at the end of the record')
 566
 567     cookie = compat_cookiejar_Cookie(
 568         version=0, name=name, value=value, port=None, port_specified=False,
 569         domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'),
 570         path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False,
 571         comment=None, comment_url=None, rest={})
 572     jar.set_cookie(cookie)
 573     return record_size
 574
 575
 576 def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
 577     """
 578     References:
 579         - https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc
 580             - this data appears to be out of date but the important parts of the database structure is the same
 581             - there are a few bytes here and there which are skipped during parsing
 582     """
 583     if jar is None:
 584         jar = YoutubeDLCookieJar()
 585     page_sizes, body_start = _parse_safari_cookies_header(data, logger)
 586     p = DataParser(data[body_start:], logger)
 587     for page_size in page_sizes:
 588         _parse_safari_cookies_page(p.read_bytes(page_size), jar, logger)
 589     p.skip_to_end('footer')
 590     return jar
 591
 592
 593 def _get_linux_keyring_password(browser_keyring_name):
 594     password = keyring.get_password('{} Keys'.format(browser_keyring_name),
 595                                     '{} Safe Storage'.format(browser_keyring_name))
 596     if password is None:
 597         # this sometimes occurs in KDE because chrome does not check hasEntry and instead
 598         # just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry
 599         # to verify this:
 600         # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
 601         # while starting chrome.
 602         # this may be a bug as the intended behaviour is to generate a random password and store
 603         # it, but that doesn't matter here.
 604         password = ''
 605     return password.encode('utf-8')
 606
 607
 608 def _get_mac_keyring_password(browser_keyring_name):
 609     if KEYRING_AVAILABLE:
 610         password = keyring.get_password('{} Safe Storage'.format(browser_keyring_name), browser_keyring_name)
 611         return password.encode('utf-8')
 612     else:
 613         proc = subprocess.Popen(['security', 'find-generic-password',
 614                                  '-w',  # write password to stdout
 615                                  '-a', browser_keyring_name,  # match 'account'
 616                                  '-s', '{} Safe Storage'.format(browser_keyring_name)],  # match 'service'
 617                                 stdout=subprocess.PIPE,
 618                                 stderr=subprocess.DEVNULL)
 619         try:
 620             stdout, stderr = process_communicate_or_kill(proc)
 621             return stdout
 622         except BaseException:
 623             return None
 624
 625
 626 def _get_windows_v10_key(browser_root, logger):
 627     path = _find_most_recently_used_file(browser_root, 'Local State')
 628     if path is None:
 629         logger.error('could not find local state file')
 630         return None
 631     with open(path, 'r') as f:
 632         data = json.load(f)
 633     try:
 634         base64_key = data['os_crypt']['encrypted_key']
 635     except KeyError:
 636         logger.error('no encrypted key in Local State')
 637         return None
 638     encrypted_key = compat_b64decode(base64_key)
 639     prefix = b'DPAPI'
 640     if not encrypted_key.startswith(prefix):
 641         logger.error('invalid key')
 642         return None
 643     return _decrypt_windows_dpapi(encrypted_key[len(prefix):], logger)
 644
 645
 646 def pbkdf2_sha1(password, salt, iterations, key_length):
 647     return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
 648
 649
 650 def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
 651     plaintext = aes_cbc_decrypt(bytes_to_intlist(ciphertext),
 652                                 bytes_to_intlist(key),
 653                                 bytes_to_intlist(initialization_vector))
 654     padding_length = plaintext[-1]
 655     try:
 656         return intlist_to_bytes(plaintext[:-padding_length]).decode('utf-8')
 657     except UnicodeDecodeError:
 658         logger.warning('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
 659         return None
 660
 661
 662 def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
 663     cipher = AES.new(key, AES.MODE_GCM, nonce)
 664     try:
 665         plaintext = cipher.decrypt_and_verify(ciphertext, authentication_tag)
 666     except ValueError:
 667         logger.warning('failed to decrypt cookie because the MAC check failed. Possibly the key is wrong?', only_once=True)
 668         return None
 669
 670     try:
 671         return plaintext.decode('utf-8')
 672     except UnicodeDecodeError:
 673         logger.warning('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
 674         return None
 675
 676
 677 def _decrypt_windows_dpapi(ciphertext, logger):
 678     """
 679     References:
 680         - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata
 681     """
 682     from ctypes.wintypes import DWORD
 683
 684     class DATA_BLOB(ctypes.Structure):
 685         _fields_ = [('cbData', DWORD),
 686                     ('pbData', ctypes.POINTER(ctypes.c_char))]
 687
 688     buffer = ctypes.create_string_buffer(ciphertext)
 689     blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer)
 690     blob_out = DATA_BLOB()
 691     ret = ctypes.windll.crypt32.CryptUnprotectData(
 692         ctypes.byref(blob_in),  # pDataIn
 693         None,  # ppszDataDescr: human readable description of pDataIn
 694         None,  # pOptionalEntropy: salt?
 695         None,  # pvReserved: must be NULL
 696         None,  # pPromptStruct: information about prompts to display
 697         0,  # dwFlags
 698         ctypes.byref(blob_out)  # pDataOut
 699     )
 700     if not ret:
 701         logger.warning('failed to decrypt with DPAPI', only_once=True)
 702         return None
 703
 704     result = ctypes.string_at(blob_out.pbData, blob_out.cbData)
 705     ctypes.windll.kernel32.LocalFree(blob_out.pbData)
 706     return result
 707
 708
 709 def _config_home():
 710     return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
 711
 712
 713 def _open_database_copy(database_path, tmpdir):
 714     # cannot open sqlite databases if they are already in use (e.g. by the browser)
 715     database_copy_path = os.path.join(tmpdir, 'temporary.sqlite')
 716     shutil.copy(database_path, database_copy_path)
 717     conn = sqlite3.connect(database_copy_path)
 718     return conn.cursor()
 719
 720
 721 def _get_column_names(cursor, table_name):
 722     table_info = cursor.execute('PRAGMA table_info({})'.format(table_name)).fetchall()
 723     return [row[1].decode('utf-8') for row in table_info]
 724
 725
 726 def _find_most_recently_used_file(root, filename):
 727     # if there are multiple browser profiles, take the most recently used one
 728     paths = []
 729     for root, dirs, files in os.walk(root):
 730         for file in files:
 731             if file == filename:
 732                 paths.append(os.path.join(root, file))
 733     return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
 734
 735
 736 def _merge_cookie_jars(jars):
 737     output_jar = YoutubeDLCookieJar()
 738     for jar in jars:
 739         for cookie in jar:
 740             output_jar.set_cookie(cookie)
 741         if jar.filename is not None:
 742             output_jar.filename = jar.filename
 743     return output_jar
 744
 745
 746 def _is_path(value):
 747     return os.path.sep in value
 748
 749
 750 def _parse_browser_specification(browser_name, profile=None):
 751     if browser_name not in SUPPORTED_BROWSERS:
 752         raise ValueError(f'unsupported browser: "{browser_name}"')
 753     if profile is not None and _is_path(profile):
 754         profile = os.path.expanduser(profile)
 755     return browser_name, profile