yt_dlp/cookies.py

   1 import ctypes
   2 import json
   3 import os
   4 import shutil
   5 import sqlite3
   6 import struct
   7 import subprocess
   8 import sys
   9 import tempfile
  10 from datetime import datetime, timedelta, timezone
  11 from hashlib import pbkdf2_hmac
  12
  13 from yt_dlp.aes import aes_cbc_decrypt
  14 from yt_dlp.compat import (
  15     compat_b64decode,
  16     compat_cookiejar_Cookie,
  17 )
  18 from yt_dlp.utils import (
  19     bytes_to_intlist,
  20     expand_path,
  21     intlist_to_bytes,
  22     process_communicate_or_kill,
  23     YoutubeDLCookieJar,
  24 )
  25
  26 try:
  27     from Crypto.Cipher import AES
  28     CRYPTO_AVAILABLE = True
  29 except ImportError:
  30     CRYPTO_AVAILABLE = False
  31
  32 try:
  33     import keyring
  34     KEYRING_AVAILABLE = True
  35 except ImportError:
  36     KEYRING_AVAILABLE = False
  37
  38
  39 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
  40 SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
  41
  42
  43 class YDLLogger:
  44     def __init__(self, ydl=None):
  45         self._ydl = ydl
  46
  47     def debug(self, message):
  48         if self._ydl:
  49             self._ydl.write_debug(message)
  50
  51     def info(self, message):
  52         if self._ydl:
  53             self._ydl.to_screen(f'[Cookies] {message}')
  54
  55     def warning(self, message, only_once=False):
  56         if self._ydl:
  57             self._ydl.report_warning(message, only_once)
  58
  59     def error(self, message):
  60         if self._ydl:
  61             self._ydl.report_error(message)
  62
  63
  64 def load_cookies(cookie_file, browser_specification, ydl):
  65     cookie_jars = []
  66     if browser_specification is not None:
  67         browser_name, profile = _parse_browser_specification(*browser_specification)
  68         cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl)))
  69
  70     if cookie_file is not None:
  71         cookie_file = expand_path(cookie_file)
  72         jar = YoutubeDLCookieJar(cookie_file)
  73         if os.access(cookie_file, os.R_OK):
  74             jar.load(ignore_discard=True, ignore_expires=True)
  75         cookie_jars.append(jar)
  76
  77     return _merge_cookie_jars(cookie_jars)
  78
  79
  80 def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger()):
  81     if browser_name == 'firefox':
  82         return _extract_firefox_cookies(profile, logger)
  83     elif browser_name == 'safari':
  84         return _extract_safari_cookies(profile, logger)
  85     elif browser_name in CHROMIUM_BASED_BROWSERS:
  86         return _extract_chrome_cookies(browser_name, profile, logger)
  87     else:
  88         raise ValueError('unknown browser: {}'.format(browser_name))
  89
  90
  91 def _extract_firefox_cookies(profile, logger):
  92     logger.info('Extracting cookies from firefox')
  93
  94     if profile is None:
  95         search_root = _firefox_browser_dir()
  96     elif _is_path(profile):
  97         search_root = profile
  98     else:
  99         search_root = os.path.join(_firefox_browser_dir(), profile)
 100
 101     cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite')
 102     if cookie_database_path is None:
 103         raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
 104     logger.debug('extracting from: "{}"'.format(cookie_database_path))
 105
 106     with tempfile.TemporaryDirectory(prefix='youtube_dl') as tmpdir:
 107         cursor = None
 108         try:
 109             cursor = _open_database_copy(cookie_database_path, tmpdir)
 110             cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
 111             jar = YoutubeDLCookieJar()
 112             for host, name, value, path, expiry, is_secure in cursor.fetchall():
 113                 cookie = compat_cookiejar_Cookie(
 114                     version=0, name=name, value=value, port=None, port_specified=False,
 115                     domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
 116                     path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
 117                     comment=None, comment_url=None, rest={})
 118                 jar.set_cookie(cookie)
 119             logger.info('Extracted {} cookies from firefox'.format(len(jar)))
 120             return jar
 121         finally:
 122             if cursor is not None:
 123                 cursor.connection.close()
 124
 125
 126 def _firefox_browser_dir():
 127     if sys.platform in ('linux', 'linux2'):
 128         return os.path.expanduser('~/.mozilla/firefox')
 129     elif sys.platform == 'win32':
 130         return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox\Profiles')
 131     elif sys.platform == 'darwin':
 132         return os.path.expanduser('~/Library/Application Support/Firefox')
 133     else:
 134         raise ValueError('unsupported platform: {}'.format(sys.platform))
 135
 136
 137 def _get_chromium_based_browser_settings(browser_name):
 138     # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md
 139     if sys.platform in ('linux', 'linux2'):
 140         config = _config_home()
 141         browser_dir = {
 142             'brave': os.path.join(config, 'BraveSoftware/Brave-Browser'),
 143             'chrome': os.path.join(config, 'google-chrome'),
 144             'chromium': os.path.join(config, 'chromium'),
 145             'edge': os.path.join(config, 'microsoft-edge'),
 146             'opera': os.path.join(config, 'opera'),
 147             'vivaldi': os.path.join(config, 'vivaldi'),
 148         }[browser_name]
 149
 150     elif sys.platform == 'win32':
 151         appdata_local = os.path.expandvars('%LOCALAPPDATA%')
 152         appdata_roaming = os.path.expandvars('%APPDATA%')
 153         browser_dir = {
 154             'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'),
 155             'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'),
 156             'chromium': os.path.join(appdata_local, r'Chromium\User Data'),
 157             'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'),
 158             'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'),
 159             'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'),
 160         }[browser_name]
 161
 162     elif sys.platform == 'darwin':
 163         appdata = os.path.expanduser('~/Library/Application Support')
 164         browser_dir = {
 165             'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'),
 166             'chrome': os.path.join(appdata, 'Google/Chrome'),
 167             'chromium': os.path.join(appdata, 'Chromium'),
 168             'edge': os.path.join(appdata, 'Microsoft Edge'),
 169             'opera': os.path.join(appdata, 'com.operasoftware.Opera'),
 170             'vivaldi': os.path.join(appdata, 'Vivaldi'),
 171         }[browser_name]
 172
 173     else:
 174         raise ValueError('unsupported platform: {}'.format(sys.platform))
 175
 176     # Linux keyring names can be determined by snooping on dbus while opening the browser in KDE:
 177     # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
 178     keyring_name = {
 179         'brave': 'Brave',
 180         'chrome': 'Chrome',
 181         'chromium': 'Chromium',
 182         'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium',
 183         'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium',
 184         'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome',
 185     }[browser_name]
 186
 187     browsers_without_profiles = {'opera'}
 188
 189     return {
 190         'browser_dir': browser_dir,
 191         'keyring_name': keyring_name,
 192         'supports_profiles': browser_name not in browsers_without_profiles
 193     }
 194
 195
 196 def _extract_chrome_cookies(browser_name, profile, logger):
 197     logger.info('Extracting cookies from {}'.format(browser_name))
 198     config = _get_chromium_based_browser_settings(browser_name)
 199
 200     if profile is None:
 201         search_root = config['browser_dir']
 202     elif _is_path(profile):
 203         search_root = profile
 204         config['browser_dir'] = os.path.dirname(profile) if config['supports_profiles'] else profile
 205     else:
 206         if config['supports_profiles']:
 207             search_root = os.path.join(config['browser_dir'], profile)
 208         else:
 209             logger.error('{} does not support profiles'.format(browser_name))
 210             search_root = config['browser_dir']
 211
 212     cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies')
 213     if cookie_database_path is None:
 214         raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
 215     logger.debug('extracting from: "{}"'.format(cookie_database_path))
 216
 217     decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger)
 218
 219     with tempfile.TemporaryDirectory(prefix='youtube_dl') as tmpdir:
 220         cursor = None
 221         try:
 222             cursor = _open_database_copy(cookie_database_path, tmpdir)
 223             cursor.connection.text_factory = bytes
 224             column_names = _get_column_names(cursor, 'cookies')
 225             secure_column = 'is_secure' if 'is_secure' in column_names else 'secure'
 226             cursor.execute('SELECT host_key, name, value, encrypted_value, path, '
 227                            'expires_utc, {} FROM cookies'.format(secure_column))
 228             jar = YoutubeDLCookieJar()
 229             failed_cookies = 0
 230             for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
 231                 host_key = host_key.decode('utf-8')
 232                 name = name.decode('utf-8')
 233                 value = value.decode('utf-8')
 234                 path = path.decode('utf-8')
 235
 236                 if not value and encrypted_value:
 237                     value = decryptor.decrypt(encrypted_value)
 238                     if value is None:
 239                         failed_cookies += 1
 240                         continue
 241
 242                 cookie = compat_cookiejar_Cookie(
 243                     version=0, name=name, value=value, port=None, port_specified=False,
 244                     domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
 245                     path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
 246                     comment=None, comment_url=None, rest={})
 247                 jar.set_cookie(cookie)
 248             if failed_cookies > 0:
 249                 failed_message = ' ({} could not be decrypted)'.format(failed_cookies)
 250             else:
 251                 failed_message = ''
 252             logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message))
 253             return jar
 254         finally:
 255             if cursor is not None:
 256                 cursor.connection.close()
 257
 258
 259 class ChromeCookieDecryptor:
 260     """
 261     Overview:
 262
 263         Linux:
 264         - cookies are either v10 or v11
 265             - v10: AES-CBC encrypted with a fixed key
 266             - v11: AES-CBC encrypted with an OS protected key (keyring)
 267             - v11 keys can be stored in various places depending on the activate desktop environment [2]
 268
 269         Mac:
 270         - cookies are either v10 or not v10
 271             - v10: AES-CBC encrypted with an OS protected key (keyring) and more key derivation iterations than linux
 272             - not v10: 'old data' stored as plaintext
 273
 274         Windows:
 275         - cookies are either v10 or not v10
 276             - v10: AES-GCM encrypted with a key which is encrypted with DPAPI
 277             - not v10: encrypted with DPAPI
 278
 279     Sources:
 280     - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
 281     - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc
 282         - KeyStorageLinux::CreateService
 283     """
 284
 285     def decrypt(self, encrypted_value):
 286         raise NotImplementedError
 287
 288
 289 def get_cookie_decryptor(browser_root, browser_keyring_name, logger):
 290     if sys.platform in ('linux', 'linux2'):
 291         return LinuxChromeCookieDecryptor(browser_keyring_name, logger)
 292     elif sys.platform == 'darwin':
 293         return MacChromeCookieDecryptor(browser_keyring_name, logger)
 294     elif sys.platform == 'win32':
 295         return WindowsChromeCookieDecryptor(browser_root, logger)
 296     else:
 297         raise NotImplementedError('Chrome cookie decryption is not supported '
 298                                   'on this platform: {}'.format(sys.platform))
 299
 300
 301 class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
 302     def __init__(self, browser_keyring_name, logger):
 303         self._logger = logger
 304         self._v10_key = self.derive_key(b'peanuts')
 305         if KEYRING_AVAILABLE:
 306             self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name))
 307         else:
 308             self._v11_key = None
 309
 310     @staticmethod
 311     def derive_key(password):
 312         # values from
 313         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
 314         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
 315
 316     def decrypt(self, encrypted_value):
 317         version = encrypted_value[:3]
 318         ciphertext = encrypted_value[3:]
 319
 320         if version == b'v10':
 321             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 322
 323         elif version == b'v11':
 324             if self._v11_key is None:
 325                 self._logger.warning('cannot decrypt cookie as the `keyring` module is not installed. '
 326                                      'Please install by running `python3 -m pip install keyring`. '
 327                                      'Note that depending on your platform, additional packages may be required '
 328                                      'to access the keyring, see  https://pypi.org/project/keyring', only_once=True)
 329                 return None
 330             return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
 331
 332         else:
 333             return None
 334
 335
 336 class MacChromeCookieDecryptor(ChromeCookieDecryptor):
 337     def __init__(self, browser_keyring_name, logger):
 338         self._logger = logger
 339         password = _get_mac_keyring_password(browser_keyring_name)
 340         self._v10_key = None if password is None else self.derive_key(password)
 341
 342     @staticmethod
 343     def derive_key(password):
 344         # values from
 345         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
 346         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
 347
 348     def decrypt(self, encrypted_value):
 349         version = encrypted_value[:3]
 350         ciphertext = encrypted_value[3:]
 351
 352         if version == b'v10':
 353             if self._v10_key is None:
 354                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
 355                 return None
 356
 357             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 358
 359         else:
 360             # other prefixes are considered 'old data' which were stored as plaintext
 361             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
 362             return encrypted_value
 363
 364
 365 class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
 366     def __init__(self, browser_root, logger):
 367         self._logger = logger
 368         self._v10_key = _get_windows_v10_key(browser_root, logger)
 369
 370     def decrypt(self, encrypted_value):
 371         version = encrypted_value[:3]
 372         ciphertext = encrypted_value[3:]
 373
 374         if version == b'v10':
 375             if self._v10_key is None:
 376                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
 377                 return None
 378             elif not CRYPTO_AVAILABLE:
 379                 self._logger.warning('cannot decrypt cookie as the `pycryptodome` module is not installed. '
 380                                      'Please install by running `python3 -m pip install pycryptodome`',
 381                                      only_once=True)
 382                 return None
 383
 384             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
 385             #   kNonceLength
 386             nonce_length = 96 // 8
 387             # boringssl
 388             #   EVP_AEAD_AES_GCM_TAG_LEN
 389             authentication_tag_length = 16
 390
 391             raw_ciphertext = ciphertext
 392             nonce = raw_ciphertext[:nonce_length]
 393             ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length]
 394             authentication_tag = raw_ciphertext[-authentication_tag_length:]
 395
 396             return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
 397
 398         else:
 399             # any other prefix means the data is DPAPI encrypted
 400             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
 401             return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8')
 402
 403
 404 def _extract_safari_cookies(profile, logger):
 405     if profile is not None:
 406         logger.error('safari does not support profiles')
 407     if sys.platform != 'darwin':
 408         raise ValueError('unsupported platform: {}'.format(sys.platform))
 409
 410     cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
 411
 412     if not os.path.isfile(cookies_path):
 413         raise FileNotFoundError('could not find safari cookies database')
 414
 415     with open(cookies_path, 'rb') as f:
 416         cookies_data = f.read()
 417
 418     jar = parse_safari_cookies(cookies_data, logger=logger)
 419     logger.info('Extracted {} cookies from safari'.format(len(jar)))
 420     return jar
 421
 422
 423 class ParserError(Exception):
 424     pass
 425
 426
 427 class DataParser:
 428     def __init__(self, data, logger):
 429         self._data = data
 430         self.cursor = 0
 431         self._logger = logger
 432
 433     def read_bytes(self, num_bytes):
 434         if num_bytes < 0:
 435             raise ParserError('invalid read of {} bytes'.format(num_bytes))
 436         end = self.cursor + num_bytes
 437         if end > len(self._data):
 438             raise ParserError('reached end of input')
 439         data = self._data[self.cursor:end]
 440         self.cursor = end
 441         return data
 442
 443     def expect_bytes(self, expected_value, message):
 444         value = self.read_bytes(len(expected_value))
 445         if value != expected_value:
 446             raise ParserError('unexpected value: {} != {} ({})'.format(value, expected_value, message))
 447
 448     def read_uint(self, big_endian=False):
 449         data_format = '>I' if big_endian else '<I'
 450         return struct.unpack(data_format, self.read_bytes(4))[0]
 451
 452     def read_double(self, big_endian=False):
 453         data_format = '>d' if big_endian else '<d'
 454         return struct.unpack(data_format, self.read_bytes(8))[0]
 455
 456     def read_cstring(self):
 457         buffer = []
 458         while True:
 459             c = self.read_bytes(1)
 460             if c == b'\x00':
 461                 return b''.join(buffer).decode('utf-8')
 462             else:
 463                 buffer.append(c)
 464
 465     def skip(self, num_bytes, description='unknown'):
 466         if num_bytes > 0:
 467             self._logger.debug('skipping {} bytes ({}): {}'.format(
 468                 num_bytes, description, self.read_bytes(num_bytes)))
 469         elif num_bytes < 0:
 470             raise ParserError('invalid skip of {} bytes'.format(num_bytes))
 471
 472     def skip_to(self, offset, description='unknown'):
 473         self.skip(offset - self.cursor, description)
 474
 475     def skip_to_end(self, description='unknown'):
 476         self.skip_to(len(self._data), description)
 477
 478
 479 def _mac_absolute_time_to_posix(timestamp):
 480     return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp())
 481
 482
 483 def _parse_safari_cookies_header(data, logger):
 484     p = DataParser(data, logger)
 485     p.expect_bytes(b'cook', 'database signature')
 486     number_of_pages = p.read_uint(big_endian=True)
 487     page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)]
 488     return page_sizes, p.cursor
 489
 490
 491 def _parse_safari_cookies_page(data, jar, logger):
 492     p = DataParser(data, logger)
 493     p.expect_bytes(b'\x00\x00\x01\x00', 'page signature')
 494     number_of_cookies = p.read_uint()
 495     record_offsets = [p.read_uint() for _ in range(number_of_cookies)]
 496     if number_of_cookies == 0:
 497         logger.debug('a cookies page of size {} has no cookies'.format(len(data)))
 498         return
 499
 500     p.skip_to(record_offsets[0], 'unknown page header field')
 501
 502     for record_offset in record_offsets:
 503         p.skip_to(record_offset, 'space between records')
 504         record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
 505         p.read_bytes(record_length)
 506     p.skip_to_end('space in between pages')
 507
 508
 509 def _parse_safari_cookies_record(data, jar, logger):
 510     p = DataParser(data, logger)
 511     record_size = p.read_uint()
 512     p.skip(4, 'unknown record field 1')
 513     flags = p.read_uint()
 514     is_secure = bool(flags & 0x0001)
 515     p.skip(4, 'unknown record field 2')
 516     domain_offset = p.read_uint()
 517     name_offset = p.read_uint()
 518     path_offset = p.read_uint()
 519     value_offset = p.read_uint()
 520     p.skip(8, 'unknown record field 3')
 521     expiration_date = _mac_absolute_time_to_posix(p.read_double())
 522     _creation_date = _mac_absolute_time_to_posix(p.read_double())  # noqa: F841
 523
 524     try:
 525         p.skip_to(domain_offset)
 526         domain = p.read_cstring()
 527
 528         p.skip_to(name_offset)
 529         name = p.read_cstring()
 530
 531         p.skip_to(path_offset)
 532         path = p.read_cstring()
 533
 534         p.skip_to(value_offset)
 535         value = p.read_cstring()
 536     except UnicodeDecodeError:
 537         logger.warning('failed to parse cookie because UTF-8 decoding failed')
 538         return record_size
 539
 540     p.skip_to(record_size, 'space at the end of the record')
 541
 542     cookie = compat_cookiejar_Cookie(
 543         version=0, name=name, value=value, port=None, port_specified=False,
 544         domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'),
 545         path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False,
 546         comment=None, comment_url=None, rest={})
 547     jar.set_cookie(cookie)
 548     return record_size
 549
 550
 551 def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
 552     """
 553     References:
 554         - https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc
 555             - this data appears to be out of date but the important parts of the database structure is the same
 556             - there are a few bytes here and there which are skipped during parsing
 557     """
 558     if jar is None:
 559         jar = YoutubeDLCookieJar()
 560     page_sizes, body_start = _parse_safari_cookies_header(data, logger)
 561     p = DataParser(data[body_start:], logger)
 562     for page_size in page_sizes:
 563         _parse_safari_cookies_page(p.read_bytes(page_size), jar, logger)
 564     p.skip_to_end('footer')
 565     return jar
 566
 567
 568 def _get_linux_keyring_password(browser_keyring_name):
 569     password = keyring.get_password('{} Keys'.format(browser_keyring_name),
 570                                     '{} Safe Storage'.format(browser_keyring_name))
 571     if password is None:
 572         # this sometimes occurs in KDE because chrome does not check hasEntry and instead
 573         # just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry
 574         # to verify this:
 575         # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
 576         # while starting chrome.
 577         # this may be a bug as the intended behaviour is to generate a random password and store
 578         # it, but that doesn't matter here.
 579         password = ''
 580     return password.encode('utf-8')
 581
 582
 583 def _get_mac_keyring_password(browser_keyring_name):
 584     if KEYRING_AVAILABLE:
 585         password = keyring.get_password('{} Safe Storage'.format(browser_keyring_name), browser_keyring_name)
 586         return password.encode('utf-8')
 587     else:
 588         proc = subprocess.Popen(['security', 'find-generic-password',
 589                                  '-w',  # write password to stdout
 590                                  '-a', browser_keyring_name,  # match 'account'
 591                                  '-s', '{} Safe Storage'.format(browser_keyring_name)],  # match 'service'
 592                                 stdout=subprocess.PIPE,
 593                                 stderr=subprocess.DEVNULL)
 594         try:
 595             stdout, stderr = process_communicate_or_kill(proc)
 596             return stdout
 597         except BaseException:
 598             return None
 599
 600
 601 def _get_windows_v10_key(browser_root, logger):
 602     path = _find_most_recently_used_file(browser_root, 'Local State')
 603     if path is None:
 604         logger.error('could not find local state file')
 605         return None
 606     with open(path, 'r') as f:
 607         data = json.load(f)
 608     try:
 609         base64_key = data['os_crypt']['encrypted_key']
 610     except KeyError:
 611         logger.error('no encrypted key in Local State')
 612         return None
 613     encrypted_key = compat_b64decode(base64_key)
 614     prefix = b'DPAPI'
 615     if not encrypted_key.startswith(prefix):
 616         logger.error('invalid key')
 617         return None
 618     return _decrypt_windows_dpapi(encrypted_key[len(prefix):], logger)
 619
 620
 621 def pbkdf2_sha1(password, salt, iterations, key_length):
 622     return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
 623
 624
 625 def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
 626     plaintext = aes_cbc_decrypt(bytes_to_intlist(ciphertext),
 627                                 bytes_to_intlist(key),
 628                                 bytes_to_intlist(initialization_vector))
 629     padding_length = plaintext[-1]
 630     try:
 631         return intlist_to_bytes(plaintext[:-padding_length]).decode('utf-8')
 632     except UnicodeDecodeError:
 633         logger.warning('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?')
 634         return None
 635
 636
 637 def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
 638     cipher = AES.new(key, AES.MODE_GCM, nonce)
 639     try:
 640         plaintext = cipher.decrypt_and_verify(ciphertext, authentication_tag)
 641     except ValueError:
 642         logger.warning('failed to decrypt cookie because the MAC check failed. Possibly the key is wrong?')
 643         return None
 644
 645     try:
 646         return plaintext.decode('utf-8')
 647     except UnicodeDecodeError:
 648         logger.warning('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?')
 649         return None
 650
 651
 652 def _decrypt_windows_dpapi(ciphertext, logger):
 653     """
 654     References:
 655         - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata
 656     """
 657     from ctypes.wintypes import DWORD
 658
 659     class DATA_BLOB(ctypes.Structure):
 660         _fields_ = [('cbData', DWORD),
 661                     ('pbData', ctypes.POINTER(ctypes.c_char))]
 662
 663     buffer = ctypes.create_string_buffer(ciphertext)
 664     blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer)
 665     blob_out = DATA_BLOB()
 666     ret = ctypes.windll.crypt32.CryptUnprotectData(
 667         ctypes.byref(blob_in),  # pDataIn
 668         None,  # ppszDataDescr: human readable description of pDataIn
 669         None,  # pOptionalEntropy: salt?
 670         None,  # pvReserved: must be NULL
 671         None,  # pPromptStruct: information about prompts to display
 672         0,  # dwFlags
 673         ctypes.byref(blob_out)  # pDataOut
 674     )
 675     if not ret:
 676         logger.warning('failed to decrypt with DPAPI')
 677         return None
 678
 679     result = ctypes.string_at(blob_out.pbData, blob_out.cbData)
 680     ctypes.windll.kernel32.LocalFree(blob_out.pbData)
 681     return result
 682
 683
 684 def _config_home():
 685     return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
 686
 687
 688 def _open_database_copy(database_path, tmpdir):
 689     # cannot open sqlite databases if they are already in use (e.g. by the browser)
 690     database_copy_path = os.path.join(tmpdir, 'temporary.sqlite')
 691     shutil.copy(database_path, database_copy_path)
 692     conn = sqlite3.connect(database_copy_path)
 693     return conn.cursor()
 694
 695
 696 def _get_column_names(cursor, table_name):
 697     table_info = cursor.execute('PRAGMA table_info({})'.format(table_name)).fetchall()
 698     return [row[1].decode('utf-8') for row in table_info]
 699
 700
 701 def _find_most_recently_used_file(root, filename):
 702     # if there are multiple browser profiles, take the most recently used one
 703     paths = []
 704     for root, dirs, files in os.walk(root):
 705         for file in files:
 706             if file == filename:
 707                 paths.append(os.path.join(root, file))
 708     return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
 709
 710
 711 def _merge_cookie_jars(jars):
 712     output_jar = YoutubeDLCookieJar()
 713     for jar in jars:
 714         for cookie in jar:
 715             output_jar.set_cookie(cookie)
 716         if jar.filename is not None:
 717             output_jar.filename = jar.filename
 718     return output_jar
 719
 720
 721 def _is_path(value):
 722     return os.path.sep in value
 723
 724
 725 def _parse_browser_specification(browser_name, profile=None):
 726     if browser_name not in SUPPORTED_BROWSERS:
 727         raise ValueError(f'unsupported browser: "{browser_name}"')
 728     if profile is not None and _is_path(profile):
 729         profile = os.path.expanduser(profile)
 730     return browser_name, profile