yt_dlp/cookies.py

   1 import ctypes
   2 import json
   3 import os
   4 import shutil
   5 import struct
   6 import subprocess
   7 import sys
   8 import tempfile
   9 from datetime import datetime, timedelta, timezone
  10 from hashlib import pbkdf2_hmac
  11
  12 from .aes import aes_cbc_decrypt_bytes, aes_gcm_decrypt_and_verify_bytes
  13 from .compat import (
  14     compat_b64decode,
  15     compat_cookiejar_Cookie,
  16 )
  17 from .utils import (
  18     bug_reports_message,
  19     expand_path,
  20     Popen,
  21     YoutubeDLCookieJar,
  22 )
  23
  24 try:
  25     import sqlite3
  26     SQLITE_AVAILABLE = True
  27 except ImportError:
  28     # although sqlite3 is part of the standard library, it is possible to compile python without
  29     # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544
  30     SQLITE_AVAILABLE = False
  31
  32
  33 try:
  34     import keyring
  35     KEYRING_AVAILABLE = True
  36     KEYRING_UNAVAILABLE_REASON = f'due to unknown reasons{bug_reports_message()}'
  37 except ImportError:
  38     KEYRING_AVAILABLE = False
  39     KEYRING_UNAVAILABLE_REASON = (
  40         'as the `keyring` module is not installed. '
  41         'Please install by running `python3 -m pip install keyring`. '
  42         'Depending on your platform, additional packages may be required '
  43         'to access the keyring; see  https://pypi.org/project/keyring')
  44 except Exception as _err:
  45     KEYRING_AVAILABLE = False
  46     KEYRING_UNAVAILABLE_REASON = 'as the `keyring` module could not be initialized: %s' % _err
  47
  48
  49 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
  50 SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
  51
  52
  53 class YDLLogger:
  54     def __init__(self, ydl=None):
  55         self._ydl = ydl
  56
  57     def debug(self, message):
  58         if self._ydl:
  59             self._ydl.write_debug(message)
  60
  61     def info(self, message):
  62         if self._ydl:
  63             self._ydl.to_screen(f'[Cookies] {message}')
  64
  65     def warning(self, message, only_once=False):
  66         if self._ydl:
  67             self._ydl.report_warning(message, only_once)
  68
  69     def error(self, message):
  70         if self._ydl:
  71             self._ydl.report_error(message)
  72
  73
  74 def load_cookies(cookie_file, browser_specification, ydl):
  75     cookie_jars = []
  76     if browser_specification is not None:
  77         browser_name, profile = _parse_browser_specification(*browser_specification)
  78         cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl)))
  79
  80     if cookie_file is not None:
  81         cookie_file = expand_path(cookie_file)
  82         jar = YoutubeDLCookieJar(cookie_file)
  83         if os.access(cookie_file, os.R_OK):
  84             jar.load(ignore_discard=True, ignore_expires=True)
  85         cookie_jars.append(jar)
  86
  87     return _merge_cookie_jars(cookie_jars)
  88
  89
  90 def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger()):
  91     if browser_name == 'firefox':
  92         return _extract_firefox_cookies(profile, logger)
  93     elif browser_name == 'safari':
  94         return _extract_safari_cookies(profile, logger)
  95     elif browser_name in CHROMIUM_BASED_BROWSERS:
  96         return _extract_chrome_cookies(browser_name, profile, logger)
  97     else:
  98         raise ValueError('unknown browser: {}'.format(browser_name))
  99
 100
 101 def _extract_firefox_cookies(profile, logger):
 102     logger.info('Extracting cookies from firefox')
 103     if not SQLITE_AVAILABLE:
 104         logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
 105                        'Please use a python interpreter compiled with sqlite3 support')
 106         return YoutubeDLCookieJar()
 107
 108     if profile is None:
 109         search_root = _firefox_browser_dir()
 110     elif _is_path(profile):
 111         search_root = profile
 112     else:
 113         search_root = os.path.join(_firefox_browser_dir(), profile)
 114
 115     cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite')
 116     if cookie_database_path is None:
 117         raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
 118     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 119
 120     with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
 121         cursor = None
 122         try:
 123             cursor = _open_database_copy(cookie_database_path, tmpdir)
 124             cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
 125             jar = YoutubeDLCookieJar()
 126             for host, name, value, path, expiry, is_secure in cursor.fetchall():
 127                 cookie = compat_cookiejar_Cookie(
 128                     version=0, name=name, value=value, port=None, port_specified=False,
 129                     domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
 130                     path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
 131                     comment=None, comment_url=None, rest={})
 132                 jar.set_cookie(cookie)
 133             logger.info('Extracted {} cookies from firefox'.format(len(jar)))
 134             return jar
 135         finally:
 136             if cursor is not None:
 137                 cursor.connection.close()
 138
 139
 140 def _firefox_browser_dir():
 141     if sys.platform in ('linux', 'linux2'):
 142         return os.path.expanduser('~/.mozilla/firefox')
 143     elif sys.platform == 'win32':
 144         return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox\Profiles')
 145     elif sys.platform == 'darwin':
 146         return os.path.expanduser('~/Library/Application Support/Firefox')
 147     else:
 148         raise ValueError('unsupported platform: {}'.format(sys.platform))
 149
 150
 151 def _get_chromium_based_browser_settings(browser_name):
 152     # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md
 153     if sys.platform in ('linux', 'linux2'):
 154         config = _config_home()
 155         browser_dir = {
 156             'brave': os.path.join(config, 'BraveSoftware/Brave-Browser'),
 157             'chrome': os.path.join(config, 'google-chrome'),
 158             'chromium': os.path.join(config, 'chromium'),
 159             'edge': os.path.join(config, 'microsoft-edge'),
 160             'opera': os.path.join(config, 'opera'),
 161             'vivaldi': os.path.join(config, 'vivaldi'),
 162         }[browser_name]
 163
 164     elif sys.platform == 'win32':
 165         appdata_local = os.path.expandvars('%LOCALAPPDATA%')
 166         appdata_roaming = os.path.expandvars('%APPDATA%')
 167         browser_dir = {
 168             'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'),
 169             'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'),
 170             'chromium': os.path.join(appdata_local, r'Chromium\User Data'),
 171             'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'),
 172             'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'),
 173             'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'),
 174         }[browser_name]
 175
 176     elif sys.platform == 'darwin':
 177         appdata = os.path.expanduser('~/Library/Application Support')
 178         browser_dir = {
 179             'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'),
 180             'chrome': os.path.join(appdata, 'Google/Chrome'),
 181             'chromium': os.path.join(appdata, 'Chromium'),
 182             'edge': os.path.join(appdata, 'Microsoft Edge'),
 183             'opera': os.path.join(appdata, 'com.operasoftware.Opera'),
 184             'vivaldi': os.path.join(appdata, 'Vivaldi'),
 185         }[browser_name]
 186
 187     else:
 188         raise ValueError('unsupported platform: {}'.format(sys.platform))
 189
 190     # Linux keyring names can be determined by snooping on dbus while opening the browser in KDE:
 191     # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
 192     keyring_name = {
 193         'brave': 'Brave',
 194         'chrome': 'Chrome',
 195         'chromium': 'Chromium',
 196         'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium',
 197         'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium',
 198         'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome',
 199     }[browser_name]
 200
 201     browsers_without_profiles = {'opera'}
 202
 203     return {
 204         'browser_dir': browser_dir,
 205         'keyring_name': keyring_name,
 206         'supports_profiles': browser_name not in browsers_without_profiles
 207     }
 208
 209
 210 def _extract_chrome_cookies(browser_name, profile, logger):
 211     logger.info('Extracting cookies from {}'.format(browser_name))
 212
 213     if not SQLITE_AVAILABLE:
 214         logger.warning(('Cannot extract cookies from {} without sqlite3 support. '
 215                         'Please use a python interpreter compiled with sqlite3 support').format(browser_name))
 216         return YoutubeDLCookieJar()
 217
 218     config = _get_chromium_based_browser_settings(browser_name)
 219
 220     if profile is None:
 221         search_root = config['browser_dir']
 222     elif _is_path(profile):
 223         search_root = profile
 224         config['browser_dir'] = os.path.dirname(profile) if config['supports_profiles'] else profile
 225     else:
 226         if config['supports_profiles']:
 227             search_root = os.path.join(config['browser_dir'], profile)
 228         else:
 229             logger.error('{} does not support profiles'.format(browser_name))
 230             search_root = config['browser_dir']
 231
 232     cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies')
 233     if cookie_database_path is None:
 234         raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
 235     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 236
 237     decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger)
 238
 239     with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
 240         cursor = None
 241         try:
 242             cursor = _open_database_copy(cookie_database_path, tmpdir)
 243             cursor.connection.text_factory = bytes
 244             column_names = _get_column_names(cursor, 'cookies')
 245             secure_column = 'is_secure' if 'is_secure' in column_names else 'secure'
 246             cursor.execute('SELECT host_key, name, value, encrypted_value, path, '
 247                            'expires_utc, {} FROM cookies'.format(secure_column))
 248             jar = YoutubeDLCookieJar()
 249             failed_cookies = 0
 250             for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
 251                 host_key = host_key.decode('utf-8')
 252                 name = name.decode('utf-8')
 253                 value = value.decode('utf-8')
 254                 path = path.decode('utf-8')
 255
 256                 if not value and encrypted_value:
 257                     value = decryptor.decrypt(encrypted_value)
 258                     if value is None:
 259                         failed_cookies += 1
 260                         continue
 261
 262                 cookie = compat_cookiejar_Cookie(
 263                     version=0, name=name, value=value, port=None, port_specified=False,
 264                     domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
 265                     path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
 266                     comment=None, comment_url=None, rest={})
 267                 jar.set_cookie(cookie)
 268             if failed_cookies > 0:
 269                 failed_message = ' ({} could not be decrypted)'.format(failed_cookies)
 270             else:
 271                 failed_message = ''
 272             logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message))
 273             return jar
 274         finally:
 275             if cursor is not None:
 276                 cursor.connection.close()
 277
 278
 279 class ChromeCookieDecryptor:
 280     """
 281     Overview:
 282
 283         Linux:
 284         - cookies are either v10 or v11
 285             - v10: AES-CBC encrypted with a fixed key
 286             - v11: AES-CBC encrypted with an OS protected key (keyring)
 287             - v11 keys can be stored in various places depending on the activate desktop environment [2]
 288
 289         Mac:
 290         - cookies are either v10 or not v10
 291             - v10: AES-CBC encrypted with an OS protected key (keyring) and more key derivation iterations than linux
 292             - not v10: 'old data' stored as plaintext
 293
 294         Windows:
 295         - cookies are either v10 or not v10
 296             - v10: AES-GCM encrypted with a key which is encrypted with DPAPI
 297             - not v10: encrypted with DPAPI
 298
 299     Sources:
 300     - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
 301     - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc
 302         - KeyStorageLinux::CreateService
 303     """
 304
 305     def decrypt(self, encrypted_value):
 306         raise NotImplementedError
 307
 308
 309 def get_cookie_decryptor(browser_root, browser_keyring_name, logger):
 310     if sys.platform in ('linux', 'linux2'):
 311         return LinuxChromeCookieDecryptor(browser_keyring_name, logger)
 312     elif sys.platform == 'darwin':
 313         return MacChromeCookieDecryptor(browser_keyring_name, logger)
 314     elif sys.platform == 'win32':
 315         return WindowsChromeCookieDecryptor(browser_root, logger)
 316     else:
 317         raise NotImplementedError('Chrome cookie decryption is not supported '
 318                                   'on this platform: {}'.format(sys.platform))
 319
 320
 321 class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
 322     def __init__(self, browser_keyring_name, logger):
 323         self._logger = logger
 324         self._v10_key = self.derive_key(b'peanuts')
 325         if KEYRING_AVAILABLE:
 326             self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name))
 327         else:
 328             self._v11_key = None
 329
 330     @staticmethod
 331     def derive_key(password):
 332         # values from
 333         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
 334         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
 335
 336     def decrypt(self, encrypted_value):
 337         version = encrypted_value[:3]
 338         ciphertext = encrypted_value[3:]
 339
 340         if version == b'v10':
 341             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 342
 343         elif version == b'v11':
 344             if self._v11_key is None:
 345                 self._logger.warning(f'cannot decrypt cookie {KEYRING_UNAVAILABLE_REASON}', only_once=True)
 346                 return None
 347             return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
 348
 349         else:
 350             return None
 351
 352
 353 class MacChromeCookieDecryptor(ChromeCookieDecryptor):
 354     def __init__(self, browser_keyring_name, logger):
 355         self._logger = logger
 356         password = _get_mac_keyring_password(browser_keyring_name, logger)
 357         self._v10_key = None if password is None else self.derive_key(password)
 358
 359     @staticmethod
 360     def derive_key(password):
 361         # values from
 362         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
 363         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
 364
 365     def decrypt(self, encrypted_value):
 366         version = encrypted_value[:3]
 367         ciphertext = encrypted_value[3:]
 368
 369         if version == b'v10':
 370             if self._v10_key is None:
 371                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
 372                 return None
 373
 374             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 375
 376         else:
 377             # other prefixes are considered 'old data' which were stored as plaintext
 378             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
 379             return encrypted_value
 380
 381
 382 class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
 383     def __init__(self, browser_root, logger):
 384         self._logger = logger
 385         self._v10_key = _get_windows_v10_key(browser_root, logger)
 386
 387     def decrypt(self, encrypted_value):
 388         version = encrypted_value[:3]
 389         ciphertext = encrypted_value[3:]
 390
 391         if version == b'v10':
 392             if self._v10_key is None:
 393                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
 394                 return None
 395
 396             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
 397             #   kNonceLength
 398             nonce_length = 96 // 8
 399             # boringssl
 400             #   EVP_AEAD_AES_GCM_TAG_LEN
 401             authentication_tag_length = 16
 402
 403             raw_ciphertext = ciphertext
 404             nonce = raw_ciphertext[:nonce_length]
 405             ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length]
 406             authentication_tag = raw_ciphertext[-authentication_tag_length:]
 407
 408             return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
 409
 410         else:
 411             # any other prefix means the data is DPAPI encrypted
 412             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
 413             return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8')
 414
 415
 416 def _extract_safari_cookies(profile, logger):
 417     if profile is not None:
 418         logger.error('safari does not support profiles')
 419     if sys.platform != 'darwin':
 420         raise ValueError('unsupported platform: {}'.format(sys.platform))
 421
 422     cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
 423
 424     if not os.path.isfile(cookies_path):
 425         raise FileNotFoundError('could not find safari cookies database')
 426
 427     with open(cookies_path, 'rb') as f:
 428         cookies_data = f.read()
 429
 430     jar = parse_safari_cookies(cookies_data, logger=logger)
 431     logger.info('Extracted {} cookies from safari'.format(len(jar)))
 432     return jar
 433
 434
 435 class ParserError(Exception):
 436     pass
 437
 438
 439 class DataParser:
 440     def __init__(self, data, logger):
 441         self._data = data
 442         self.cursor = 0
 443         self._logger = logger
 444
 445     def read_bytes(self, num_bytes):
 446         if num_bytes < 0:
 447             raise ParserError('invalid read of {} bytes'.format(num_bytes))
 448         end = self.cursor + num_bytes
 449         if end > len(self._data):
 450             raise ParserError('reached end of input')
 451         data = self._data[self.cursor:end]
 452         self.cursor = end
 453         return data
 454
 455     def expect_bytes(self, expected_value, message):
 456         value = self.read_bytes(len(expected_value))
 457         if value != expected_value:
 458             raise ParserError('unexpected value: {} != {} ({})'.format(value, expected_value, message))
 459
 460     def read_uint(self, big_endian=False):
 461         data_format = '>I' if big_endian else '<I'
 462         return struct.unpack(data_format, self.read_bytes(4))[0]
 463
 464     def read_double(self, big_endian=False):
 465         data_format = '>d' if big_endian else '<d'
 466         return struct.unpack(data_format, self.read_bytes(8))[0]
 467
 468     def read_cstring(self):
 469         buffer = []
 470         while True:
 471             c = self.read_bytes(1)
 472             if c == b'\x00':
 473                 return b''.join(buffer).decode('utf-8')
 474             else:
 475                 buffer.append(c)
 476
 477     def skip(self, num_bytes, description='unknown'):
 478         if num_bytes > 0:
 479             self._logger.debug('skipping {} bytes ({}): {}'.format(
 480                 num_bytes, description, self.read_bytes(num_bytes)))
 481         elif num_bytes < 0:
 482             raise ParserError('invalid skip of {} bytes'.format(num_bytes))
 483
 484     def skip_to(self, offset, description='unknown'):
 485         self.skip(offset - self.cursor, description)
 486
 487     def skip_to_end(self, description='unknown'):
 488         self.skip_to(len(self._data), description)
 489
 490
 491 def _mac_absolute_time_to_posix(timestamp):
 492     return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp())
 493
 494
 495 def _parse_safari_cookies_header(data, logger):
 496     p = DataParser(data, logger)
 497     p.expect_bytes(b'cook', 'database signature')
 498     number_of_pages = p.read_uint(big_endian=True)
 499     page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)]
 500     return page_sizes, p.cursor
 501
 502
 503 def _parse_safari_cookies_page(data, jar, logger):
 504     p = DataParser(data, logger)
 505     p.expect_bytes(b'\x00\x00\x01\x00', 'page signature')
 506     number_of_cookies = p.read_uint()
 507     record_offsets = [p.read_uint() for _ in range(number_of_cookies)]
 508     if number_of_cookies == 0:
 509         logger.debug('a cookies page of size {} has no cookies'.format(len(data)))
 510         return
 511
 512     p.skip_to(record_offsets[0], 'unknown page header field')
 513
 514     for record_offset in record_offsets:
 515         p.skip_to(record_offset, 'space between records')
 516         record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
 517         p.read_bytes(record_length)
 518     p.skip_to_end('space in between pages')
 519
 520
 521 def _parse_safari_cookies_record(data, jar, logger):
 522     p = DataParser(data, logger)
 523     record_size = p.read_uint()
 524     p.skip(4, 'unknown record field 1')
 525     flags = p.read_uint()
 526     is_secure = bool(flags & 0x0001)
 527     p.skip(4, 'unknown record field 2')
 528     domain_offset = p.read_uint()
 529     name_offset = p.read_uint()
 530     path_offset = p.read_uint()
 531     value_offset = p.read_uint()
 532     p.skip(8, 'unknown record field 3')
 533     expiration_date = _mac_absolute_time_to_posix(p.read_double())
 534     _creation_date = _mac_absolute_time_to_posix(p.read_double())  # noqa: F841
 535
 536     try:
 537         p.skip_to(domain_offset)
 538         domain = p.read_cstring()
 539
 540         p.skip_to(name_offset)
 541         name = p.read_cstring()
 542
 543         p.skip_to(path_offset)
 544         path = p.read_cstring()
 545
 546         p.skip_to(value_offset)
 547         value = p.read_cstring()
 548     except UnicodeDecodeError:
 549         logger.warning('failed to parse Safari cookie because UTF-8 decoding failed', only_once=True)
 550         return record_size
 551
 552     p.skip_to(record_size, 'space at the end of the record')
 553
 554     cookie = compat_cookiejar_Cookie(
 555         version=0, name=name, value=value, port=None, port_specified=False,
 556         domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'),
 557         path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False,
 558         comment=None, comment_url=None, rest={})
 559     jar.set_cookie(cookie)
 560     return record_size
 561
 562
 563 def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
 564     """
 565     References:
 566         - https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc
 567             - this data appears to be out of date but the important parts of the database structure is the same
 568             - there are a few bytes here and there which are skipped during parsing
 569     """
 570     if jar is None:
 571         jar = YoutubeDLCookieJar()
 572     page_sizes, body_start = _parse_safari_cookies_header(data, logger)
 573     p = DataParser(data[body_start:], logger)
 574     for page_size in page_sizes:
 575         _parse_safari_cookies_page(p.read_bytes(page_size), jar, logger)
 576     p.skip_to_end('footer')
 577     return jar
 578
 579
 580 def _get_linux_keyring_password(browser_keyring_name):
 581     password = keyring.get_password('{} Keys'.format(browser_keyring_name),
 582                                     '{} Safe Storage'.format(browser_keyring_name))
 583     if password is None:
 584         # this sometimes occurs in KDE because chrome does not check hasEntry and instead
 585         # just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry
 586         # to verify this:
 587         # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
 588         # while starting chrome.
 589         # this may be a bug as the intended behaviour is to generate a random password and store
 590         # it, but that doesn't matter here.
 591         password = ''
 592     return password.encode('utf-8')
 593
 594
 595 def _get_mac_keyring_password(browser_keyring_name, logger):
 596     if KEYRING_AVAILABLE:
 597         logger.debug('using keyring to obtain password')
 598         password = keyring.get_password('{} Safe Storage'.format(browser_keyring_name), browser_keyring_name)
 599         return password.encode('utf-8')
 600     else:
 601         logger.debug('using find-generic-password to obtain password')
 602         proc = Popen(
 603             ['security', 'find-generic-password',
 604              '-w',  # write password to stdout
 605              '-a', browser_keyring_name,  # match 'account'
 606              '-s', '{} Safe Storage'.format(browser_keyring_name)],  # match 'service'
 607             stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
 608         try:
 609             stdout, stderr = proc.communicate_or_kill()
 610             if stdout[-1:] == b'\n':
 611                 stdout = stdout[:-1]
 612             return stdout
 613         except BaseException as e:
 614             logger.warning(f'exception running find-generic-password: {type(e).__name__}({e})')
 615             return None
 616
 617
 618 def _get_windows_v10_key(browser_root, logger):
 619     path = _find_most_recently_used_file(browser_root, 'Local State')
 620     if path is None:
 621         logger.error('could not find local state file')
 622         return None
 623     with open(path, 'r', encoding='utf8') as f:
 624         data = json.load(f)
 625     try:
 626         base64_key = data['os_crypt']['encrypted_key']
 627     except KeyError:
 628         logger.error('no encrypted key in Local State')
 629         return None
 630     encrypted_key = compat_b64decode(base64_key)
 631     prefix = b'DPAPI'
 632     if not encrypted_key.startswith(prefix):
 633         logger.error('invalid key')
 634         return None
 635     return _decrypt_windows_dpapi(encrypted_key[len(prefix):], logger)
 636
 637
 638 def pbkdf2_sha1(password, salt, iterations, key_length):
 639     return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
 640
 641
 642 def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
 643     plaintext = aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector)
 644     padding_length = plaintext[-1]
 645     try:
 646         return plaintext[:-padding_length].decode('utf-8')
 647     except UnicodeDecodeError:
 648         logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
 649         return None
 650
 651
 652 def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
 653     try:
 654         plaintext = aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce)
 655     except ValueError:
 656         logger.warning('failed to decrypt cookie (AES-GCM) because the MAC check failed. Possibly the key is wrong?', only_once=True)
 657         return None
 658
 659     try:
 660         return plaintext.decode('utf-8')
 661     except UnicodeDecodeError:
 662         logger.warning('failed to decrypt cookie (AES-GCM) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
 663         return None
 664
 665
 666 def _decrypt_windows_dpapi(ciphertext, logger):
 667     """
 668     References:
 669         - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata
 670     """
 671     from ctypes.wintypes import DWORD
 672
 673     class DATA_BLOB(ctypes.Structure):
 674         _fields_ = [('cbData', DWORD),
 675                     ('pbData', ctypes.POINTER(ctypes.c_char))]
 676
 677     buffer = ctypes.create_string_buffer(ciphertext)
 678     blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer)
 679     blob_out = DATA_BLOB()
 680     ret = ctypes.windll.crypt32.CryptUnprotectData(
 681         ctypes.byref(blob_in),  # pDataIn
 682         None,  # ppszDataDescr: human readable description of pDataIn
 683         None,  # pOptionalEntropy: salt?
 684         None,  # pvReserved: must be NULL
 685         None,  # pPromptStruct: information about prompts to display
 686         0,  # dwFlags
 687         ctypes.byref(blob_out)  # pDataOut
 688     )
 689     if not ret:
 690         logger.warning('failed to decrypt with DPAPI', only_once=True)
 691         return None
 692
 693     result = ctypes.string_at(blob_out.pbData, blob_out.cbData)
 694     ctypes.windll.kernel32.LocalFree(blob_out.pbData)
 695     return result
 696
 697
 698 def _config_home():
 699     return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
 700
 701
 702 def _open_database_copy(database_path, tmpdir):
 703     # cannot open sqlite databases if they are already in use (e.g. by the browser)
 704     database_copy_path = os.path.join(tmpdir, 'temporary.sqlite')
 705     shutil.copy(database_path, database_copy_path)
 706     conn = sqlite3.connect(database_copy_path)
 707     return conn.cursor()
 708
 709
 710 def _get_column_names(cursor, table_name):
 711     table_info = cursor.execute('PRAGMA table_info({})'.format(table_name)).fetchall()
 712     return [row[1].decode('utf-8') for row in table_info]
 713
 714
 715 def _find_most_recently_used_file(root, filename):
 716     # if there are multiple browser profiles, take the most recently used one
 717     paths = []
 718     for root, dirs, files in os.walk(root):
 719         for file in files:
 720             if file == filename:
 721                 paths.append(os.path.join(root, file))
 722     return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
 723
 724
 725 def _merge_cookie_jars(jars):
 726     output_jar = YoutubeDLCookieJar()
 727     for jar in jars:
 728         for cookie in jar:
 729             output_jar.set_cookie(cookie)
 730         if jar.filename is not None:
 731             output_jar.filename = jar.filename
 732     return output_jar
 733
 734
 735 def _is_path(value):
 736     return os.path.sep in value
 737
 738
 739 def _parse_browser_specification(browser_name, profile=None):
 740     browser_name = browser_name.lower()
 741     if browser_name not in SUPPORTED_BROWSERS:
 742         raise ValueError(f'unsupported browser: "{browser_name}"')
 743     if profile is not None and _is_path(profile):
 744         profile = os.path.expanduser(profile)
 745     return browser_name, profile