yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import collections
   4 import copy
   5 import datetime as dt
   6 import enum
   7 import hashlib
   8 import itertools
   9 import json
  10 import math
  11 import os.path
  12 import random
  13 import re
  14 import shlex
  15 import sys
  16 import threading
  17 import time
  18 import traceback
  19 import urllib.parse
  20
  21 from .common import InfoExtractor, SearchInfoExtractor
  22 from .openload import PhantomJSwrapper
  23 from ..compat import functools
  24 from ..jsinterp import JSInterpreter
  25 from ..networking.exceptions import HTTPError, network_exceptions
  26 from ..utils import (
  27     NO_DEFAULT,
  28     ExtractorError,
  29     LazyList,
  30     UserNotLive,
  31     bug_reports_message,
  32     classproperty,
  33     clean_html,
  34     datetime_from_str,
  35     dict_get,
  36     filesize_from_tbr,
  37     filter_dict,
  38     float_or_none,
  39     format_field,
  40     get_first,
  41     int_or_none,
  42     is_html,
  43     join_nonempty,
  44     js_to_json,
  45     mimetype2ext,
  46     orderedSet,
  47     parse_codecs,
  48     parse_count,
  49     parse_duration,
  50     parse_iso8601,
  51     parse_qs,
  52     qualities,
  53     remove_start,
  54     smuggle_url,
  55     str_or_none,
  56     str_to_int,
  57     strftime_or_none,
  58     traverse_obj,
  59     try_call,
  60     try_get,
  61     unescapeHTML,
  62     unified_strdate,
  63     unified_timestamp,
  64     unsmuggle_url,
  65     update_url_query,
  66     url_or_none,
  67     urljoin,
  68     variadic,
  69 )
  70
  71 STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
  72 # any clients starting with _ cannot be explicitly requested by the user
  73 INNERTUBE_CLIENTS = {
  74     'web': {
  75         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  76         'INNERTUBE_CONTEXT': {
  77             'client': {
  78                 'clientName': 'WEB',
  79                 'clientVersion': '2.20220801.00.00',
  80             }
  81         },
  82         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  83     },
  84     'web_embedded': {
  85         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  86         'INNERTUBE_CONTEXT': {
  87             'client': {
  88                 'clientName': 'WEB_EMBEDDED_PLAYER',
  89                 'clientVersion': '1.20220731.00.00',
  90             },
  91         },
  92         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  93     },
  94     'web_music': {
  95         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  96         'INNERTUBE_HOST': 'music.youtube.com',
  97         'INNERTUBE_CONTEXT': {
  98             'client': {
  99                 'clientName': 'WEB_REMIX',
 100                 'clientVersion': '1.20220727.01.00',
 101             }
 102         },
 103         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 104     },
 105     'web_creator': {
 106         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 107         'INNERTUBE_CONTEXT': {
 108             'client': {
 109                 'clientName': 'WEB_CREATOR',
 110                 'clientVersion': '1.20220726.00.00',
 111             }
 112         },
 113         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 114     },
 115     'android': {
 116         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 117         'INNERTUBE_CONTEXT': {
 118             'client': {
 119                 'clientName': 'ANDROID',
 120                 'clientVersion': '19.09.37',
 121                 'androidSdkVersion': 30,
 122                 'userAgent': 'com.google.android.youtube/19.09.37 (Linux; U; Android 11) gzip'
 123             }
 124         },
 125         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 126         'REQUIRE_JS_PLAYER': False
 127     },
 128     'android_embedded': {
 129         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 130         'INNERTUBE_CONTEXT': {
 131             'client': {
 132                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 133                 'clientVersion': '19.09.37',
 134                 'androidSdkVersion': 30,
 135                 'userAgent': 'com.google.android.youtube/19.09.37 (Linux; U; Android 11) gzip'
 136             },
 137         },
 138         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 139         'REQUIRE_JS_PLAYER': False
 140     },
 141     'android_music': {
 142         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 143         'INNERTUBE_CONTEXT': {
 144             'client': {
 145                 'clientName': 'ANDROID_MUSIC',
 146                 'clientVersion': '6.42.52',
 147                 'androidSdkVersion': 30,
 148                 'userAgent': 'com.google.android.apps.youtube.music/6.42.52 (Linux; U; Android 11) gzip'
 149             }
 150         },
 151         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 152         'REQUIRE_JS_PLAYER': False
 153     },
 154     'android_creator': {
 155         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 156         'INNERTUBE_CONTEXT': {
 157             'client': {
 158                 'clientName': 'ANDROID_CREATOR',
 159                 'clientVersion': '22.30.100',
 160                 'androidSdkVersion': 30,
 161                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 162             },
 163         },
 164         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 165         'REQUIRE_JS_PLAYER': False
 166     },
 167     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 168     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 169     'ios': {
 170         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 171         'INNERTUBE_CONTEXT': {
 172             'client': {
 173                 'clientName': 'IOS',
 174                 'clientVersion': '19.09.3',
 175                 'deviceModel': 'iPhone14,3',
 176                 'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 177             }
 178         },
 179         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 180         'REQUIRE_JS_PLAYER': False
 181     },
 182     'ios_embedded': {
 183         'INNERTUBE_CONTEXT': {
 184             'client': {
 185                 'clientName': 'IOS_MESSAGES_EXTENSION',
 186                 'clientVersion': '19.09.3',
 187                 'deviceModel': 'iPhone14,3',
 188                 'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 189             },
 190         },
 191         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 192         'REQUIRE_JS_PLAYER': False
 193     },
 194     'ios_music': {
 195         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 196         'INNERTUBE_CONTEXT': {
 197             'client': {
 198                 'clientName': 'IOS_MUSIC',
 199                 'clientVersion': '6.33.3',
 200                 'deviceModel': 'iPhone14,3',
 201                 'userAgent': 'com.google.ios.youtubemusic/6.33.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 202             },
 203         },
 204         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 205         'REQUIRE_JS_PLAYER': False
 206     },
 207     'ios_creator': {
 208         'INNERTUBE_CONTEXT': {
 209             'client': {
 210                 'clientName': 'IOS_CREATOR',
 211                 'clientVersion': '22.33.101',
 212                 'deviceModel': 'iPhone14,3',
 213                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 214             },
 215         },
 216         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 217         'REQUIRE_JS_PLAYER': False
 218     },
 219     # mweb has 'ultralow' formats
 220     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 221     'mweb': {
 222         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 223         'INNERTUBE_CONTEXT': {
 224             'client': {
 225                 'clientName': 'MWEB',
 226                 'clientVersion': '2.20220801.00.00',
 227             }
 228         },
 229         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 230     },
 231     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 232     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 233     'tv_embedded': {
 234         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 235         'INNERTUBE_CONTEXT': {
 236             'client': {
 237                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 238                 'clientVersion': '2.0',
 239             },
 240         },
 241         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 242     },
 243     # This client has pre-merged video+audio 720p/1080p streams
 244     'mediaconnect': {
 245         'INNERTUBE_CONTEXT': {
 246             'client': {
 247                 'clientName': 'MEDIA_CONNECT_FRONTEND',
 248                 'clientVersion': '0.1',
 249             },
 250         },
 251         'INNERTUBE_CONTEXT_CLIENT_NAME': 95
 252     },
 253 }
 254
 255
 256 def _split_innertube_client(client_name):
 257     variant, *base = client_name.rsplit('.', 1)
 258     if base:
 259         return variant, base[0], variant
 260     base, *variant = client_name.split('_', 1)
 261     return client_name, base, variant[0] if variant else None
 262
 263
 264 def short_client_name(client_name):
 265     main, *parts = _split_innertube_client(client_name)[0].replace('embedscreen', 'e_s').split('_')
 266     return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
 267
 268
 269 def build_innertube_clients():
 270     THIRD_PARTY = {
 271         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 272     }
 273     BASE_CLIENTS = ('ios', 'android', 'web', 'tv', 'mweb')
 274     priority = qualities(BASE_CLIENTS[::-1])
 275
 276     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 277         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 278         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 279         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 280         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 281
 282         _, base_client, variant = _split_innertube_client(client)
 283         ytcfg['priority'] = 10 * priority(base_client)
 284
 285         if not variant:
 286             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 287             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 288             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 289             embedscreen['priority'] -= 3
 290         elif variant == 'embedded':
 291             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 292             ytcfg['priority'] -= 2
 293         else:
 294             ytcfg['priority'] -= 3
 295
 296
 297 build_innertube_clients()
 298
 299
 300 class BadgeType(enum.Enum):
 301     AVAILABILITY_UNLISTED = enum.auto()
 302     AVAILABILITY_PRIVATE = enum.auto()
 303     AVAILABILITY_PUBLIC = enum.auto()
 304     AVAILABILITY_PREMIUM = enum.auto()
 305     AVAILABILITY_SUBSCRIPTION = enum.auto()
 306     LIVE_NOW = enum.auto()
 307     VERIFIED = enum.auto()
 308
 309
 310 class YoutubeBaseInfoExtractor(InfoExtractor):
 311     """Provide base functions for Youtube extractors"""
 312
 313     _RESERVED_NAMES = (
 314         r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
 315         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 316         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 317         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 318
 319     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 320
 321     # _NETRC_MACHINE = 'youtube'
 322
 323     # If True it will raise an error if no login info is provided
 324     _LOGIN_REQUIRED = False
 325
 326     _INVIDIOUS_SITES = (
 327         # invidious-redirect websites
 328         r'(?:www\.)?redirect\.invidious\.io',
 329         r'(?:(?:www|dev)\.)?invidio\.us',
 330         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 331         r'(?:www\.)?invidious\.pussthecat\.org',
 332         r'(?:www\.)?invidious\.zee\.li',
 333         r'(?:www\.)?invidious\.ethibox\.fr',
 334         r'(?:www\.)?iv\.ggtyler\.dev',
 335         r'(?:www\.)?inv\.vern\.i2p',
 336         r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
 337         r'(?:www\.)?inv\.riverside\.rocks',
 338         r'(?:www\.)?invidious\.silur\.me',
 339         r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
 340         r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
 341         r'(?:www\.)?invidious\.slipfox\.xyz',
 342         r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
 343         r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
 344         r'(?:www\.)?invidious\.tiekoetter\.com',
 345         r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
 346         r'(?:www\.)?invidious\.nerdvpn\.de',
 347         r'(?:www\.)?invidious\.weblibre\.org',
 348         r'(?:www\.)?inv\.odyssey346\.dev',
 349         r'(?:www\.)?invidious\.dhusch\.de',
 350         r'(?:www\.)?iv\.melmac\.space',
 351         r'(?:www\.)?watch\.thekitty\.zone',
 352         r'(?:www\.)?invidious\.privacydev\.net',
 353         r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
 354         r'(?:www\.)?invidious\.drivet\.xyz',
 355         r'(?:www\.)?vid\.priv\.au',
 356         r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
 357         r'(?:www\.)?inv\.vern\.cc',
 358         r'(?:www\.)?invidious\.esmailelbob\.xyz',
 359         r'(?:www\.)?invidious\.sethforprivacy\.com',
 360         r'(?:www\.)?yt\.oelrichsgarcia\.de',
 361         r'(?:www\.)?yt\.artemislena\.eu',
 362         r'(?:www\.)?invidious\.flokinet\.to',
 363         r'(?:www\.)?invidious\.baczek\.me',
 364         r'(?:www\.)?y\.com\.sb',
 365         r'(?:www\.)?invidious\.epicsite\.xyz',
 366         r'(?:www\.)?invidious\.lidarshield\.cloud',
 367         r'(?:www\.)?yt\.funami\.tech',
 368         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 369         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 370         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 371         # youtube-dl invidious instances list
 372         r'(?:(?:www|no)\.)?invidiou\.sh',
 373         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 374         r'(?:www\.)?invidious\.kabi\.tk',
 375         r'(?:www\.)?invidious\.mastodon\.host',
 376         r'(?:www\.)?invidious\.zapashcanon\.fr',
 377         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 378         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 379         r'(?:www\.)?invidious\.himiko\.cloud',
 380         r'(?:www\.)?invidious\.reallyancient\.tech',
 381         r'(?:www\.)?invidious\.tube',
 382         r'(?:www\.)?invidiou\.site',
 383         r'(?:www\.)?invidious\.site',
 384         r'(?:www\.)?invidious\.xyz',
 385         r'(?:www\.)?invidious\.nixnet\.xyz',
 386         r'(?:www\.)?invidious\.048596\.xyz',
 387         r'(?:www\.)?invidious\.drycat\.fr',
 388         r'(?:www\.)?inv\.skyn3t\.in',
 389         r'(?:www\.)?tube\.poal\.co',
 390         r'(?:www\.)?tube\.connect\.cafe',
 391         r'(?:www\.)?vid\.wxzm\.sx',
 392         r'(?:www\.)?vid\.mint\.lgbt',
 393         r'(?:www\.)?vid\.puffyan\.us',
 394         r'(?:www\.)?yewtu\.be',
 395         r'(?:www\.)?yt\.elukerio\.org',
 396         r'(?:www\.)?yt\.lelux\.fi',
 397         r'(?:www\.)?invidious\.ggc-project\.de',
 398         r'(?:www\.)?yt\.maisputain\.ovh',
 399         r'(?:www\.)?ytprivate\.com',
 400         r'(?:www\.)?invidious\.13ad\.de',
 401         r'(?:www\.)?invidious\.toot\.koeln',
 402         r'(?:www\.)?invidious\.fdn\.fr',
 403         r'(?:www\.)?watch\.nettohikari\.com',
 404         r'(?:www\.)?invidious\.namazso\.eu',
 405         r'(?:www\.)?invidious\.silkky\.cloud',
 406         r'(?:www\.)?invidious\.exonip\.de',
 407         r'(?:www\.)?invidious\.riverside\.rocks',
 408         r'(?:www\.)?invidious\.blamefran\.net',
 409         r'(?:www\.)?invidious\.moomoo\.de',
 410         r'(?:www\.)?ytb\.trom\.tf',
 411         r'(?:www\.)?yt\.cyberhost\.uk',
 412         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 413         r'(?:www\.)?qklhadlycap4cnod\.onion',
 414         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 415         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 416         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 417         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 418         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 419         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 420         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 421         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 422         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 423         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 424         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 425         r'(?:www\.)?piped\.kavin\.rocks',
 426         r'(?:www\.)?piped\.tokhmi\.xyz',
 427         r'(?:www\.)?piped\.syncpundit\.io',
 428         r'(?:www\.)?piped\.mha\.fi',
 429         r'(?:www\.)?watch\.whatever\.social',
 430         r'(?:www\.)?piped\.garudalinux\.org',
 431         r'(?:www\.)?piped\.rivo\.lol',
 432         r'(?:www\.)?piped-libre\.kavin\.rocks',
 433         r'(?:www\.)?yt\.jae\.fi',
 434         r'(?:www\.)?piped\.mint\.lgbt',
 435         r'(?:www\.)?il\.ax',
 436         r'(?:www\.)?piped\.esmailelbob\.xyz',
 437         r'(?:www\.)?piped\.projectsegfau\.lt',
 438         r'(?:www\.)?piped\.privacydev\.net',
 439         r'(?:www\.)?piped\.palveluntarjoaja\.eu',
 440         r'(?:www\.)?piped\.smnz\.de',
 441         r'(?:www\.)?piped\.adminforge\.de',
 442         r'(?:www\.)?watch\.whatevertinfoil\.de',
 443         r'(?:www\.)?piped\.qdi\.fi',
 444         r'(?:(?:www|cf)\.)?piped\.video',
 445         r'(?:www\.)?piped\.aeong\.one',
 446         r'(?:www\.)?piped\.moomoo\.me',
 447         r'(?:www\.)?piped\.chauvet\.pro',
 448         r'(?:www\.)?watch\.leptons\.xyz',
 449         r'(?:www\.)?pd\.vern\.cc',
 450         r'(?:www\.)?piped\.hostux\.net',
 451         r'(?:www\.)?piped\.lunar\.icu',
 452         # Hyperpipe instances from https://hyperpipe.codeberg.page/
 453         r'(?:www\.)?hyperpipe\.surge\.sh',
 454         r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
 455         r'(?:www\.)?listen\.whatever\.social',
 456         r'(?:www\.)?music\.adminforge\.de',
 457     )
 458
 459     # extracted from account/account_menu ep
 460     # XXX: These are the supported YouTube UI and API languages,
 461     # which is slightly different from languages supported for translation in YouTube studio
 462     _SUPPORTED_LANG_CODES = [
 463         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 464         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 465         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 466         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 467         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 468         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 469     ]
 470
 471     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 472
 473     _YT_HANDLE_RE = r'@[\w.-]{3,30}'  # https://support.google.com/youtube/answer/11585688?hl=en
 474     _YT_CHANNEL_UCID_RE = r'UC[\w-]{22}'
 475
 476     def ucid_or_none(self, ucid):
 477         return self._search_regex(rf'^({self._YT_CHANNEL_UCID_RE})$', ucid, 'UC-id', default=None)
 478
 479     def handle_or_none(self, handle):
 480         return self._search_regex(rf'^({self._YT_HANDLE_RE})$', handle, '@-handle', default=None)
 481
 482     def handle_from_url(self, url):
 483         return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_HANDLE_RE})',
 484                                   url, 'channel handle', default=None)
 485
 486     def ucid_from_url(self, url):
 487         return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_CHANNEL_UCID_RE})',
 488                                   url, 'channel id', default=None)
 489
 490     @functools.cached_property
 491     def _preferred_lang(self):
 492         """
 493         Returns a language code supported by YouTube for the user preferred language.
 494         Returns None if no preferred language set.
 495         """
 496         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 497         if not preferred_lang:
 498             return
 499         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 500             raise ExtractorError(
 501                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 502                 expected=True)
 503         elif preferred_lang != 'en':
 504             self.report_warning(
 505                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 506         return preferred_lang
 507
 508     def _initialize_consent(self):
 509         cookies = self._get_cookies('https://www.youtube.com/')
 510         if cookies.get('__Secure-3PSID'):
 511             return
 512         socs = cookies.get('SOCS')
 513         if socs and not socs.value.startswith('CAA'):  # not consented
 514             return
 515         self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True)  # accept all (required for mixes)
 516
 517     def _initialize_pref(self):
 518         cookies = self._get_cookies('https://www.youtube.com/')
 519         pref_cookie = cookies.get('PREF')
 520         pref = {}
 521         if pref_cookie:
 522             try:
 523                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 524             except ValueError:
 525                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 526         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 527         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 528
 529     def _real_initialize(self):
 530         self._initialize_pref()
 531         self._initialize_consent()
 532         self._check_login_required()
 533
 534     def _check_login_required(self):
 535         if self._LOGIN_REQUIRED and not self._cookies_passed:
 536             self.raise_login_required('Login details are needed to download this content', method='cookies')
 537
 538     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 539     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 540
 541     def _get_default_ytcfg(self, client='web'):
 542         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 543
 544     def _get_innertube_host(self, client='web'):
 545         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 546
 547     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 548         # try_get but with fallback to default ytcfg client values when present
 549         _func = lambda y: try_get(y, getter, expected_type)
 550         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 551
 552     def _extract_client_name(self, ytcfg, default_client='web'):
 553         return self._ytcfg_get_safe(
 554             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 555                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 556
 557     def _extract_client_version(self, ytcfg, default_client='web'):
 558         return self._ytcfg_get_safe(
 559             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 560                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 561
 562     def _select_api_hostname(self, req_api_hostname, default_client=None):
 563         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 564                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 565
 566     def _extract_api_key(self, ytcfg=None, default_client='web'):
 567         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 568
 569     def _extract_context(self, ytcfg=None, default_client='web'):
 570         context = get_first(
 571             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 572         # Enforce language and tz for extraction
 573         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 574         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 575         return context
 576
 577     _SAPISID = None
 578
 579     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 580         time_now = round(time.time())
 581         if self._SAPISID is None:
 582             yt_cookies = self._get_cookies('https://www.youtube.com')
 583             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 584             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 585             sapisid_cookie = dict_get(
 586                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 587             if sapisid_cookie and sapisid_cookie.value:
 588                 self._SAPISID = sapisid_cookie.value
 589                 self.write_debug('Extracted SAPISID cookie')
 590                 # SAPISID cookie is required if not already present
 591                 if not yt_cookies.get('SAPISID'):
 592                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 593                     self._set_cookie(
 594                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 595             else:
 596                 self._SAPISID = False
 597         if not self._SAPISID:
 598             return None
 599         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 600         sapisidhash = hashlib.sha1(
 601             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 602         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 603
 604     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 605                   note='Downloading API JSON', errnote='Unable to download API page',
 606                   context=None, api_key=None, api_hostname=None, default_client='web'):
 607
 608         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 609         data.update(query)
 610         real_headers = self.generate_api_headers(default_client=default_client)
 611         real_headers.update({'content-type': 'application/json'})
 612         if headers:
 613             real_headers.update(headers)
 614         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 615                    or api_key or self._extract_api_key(default_client=default_client))
 616         return self._download_json(
 617             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 618             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 619             data=json.dumps(data).encode('utf8'), headers=real_headers,
 620             query={'key': api_key, 'prettyPrint': 'false'})
 621
 622     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 623         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 624
 625     @staticmethod
 626     def _extract_session_index(*data):
 627         """
 628         Index of current account in account list.
 629         See: https://github.com/yt-dlp/yt-dlp/pull/519
 630         """
 631         for ytcfg in data:
 632             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 633             if session_index is not None:
 634                 return session_index
 635
 636     # Deprecated?
 637     def _extract_identity_token(self, ytcfg=None, webpage=None):
 638         if ytcfg:
 639             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 640             if token:
 641                 return token
 642         if webpage:
 643             return self._search_regex(
 644                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 645                 'identity token', default=None, fatal=False)
 646
 647     @staticmethod
 648     def _extract_account_syncid(*args):
 649         """
 650         Extract syncId required to download private playlists of secondary channels
 651         @params response and/or ytcfg
 652         """
 653         for data in args:
 654             # ytcfg includes channel_syncid if on secondary channel
 655             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 656             if delegated_sid:
 657                 return delegated_sid
 658             sync_ids = (try_get(
 659                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 660                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 661             if len(sync_ids) >= 2 and sync_ids[1]:
 662                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 663                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 664                 return sync_ids[0]
 665
 666     @staticmethod
 667     def _extract_visitor_data(*args):
 668         """
 669         Extracts visitorData from an API response or ytcfg
 670         Appears to be used to track session state
 671         """
 672         return get_first(
 673             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 674             expected_type=str)
 675
 676     @functools.cached_property
 677     def is_authenticated(self):
 678         return bool(self._generate_sapisidhash_header())
 679
 680     def extract_ytcfg(self, video_id, webpage):
 681         if not webpage:
 682             return {}
 683         return self._parse_json(
 684             self._search_regex(
 685                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 686                 default='{}'), video_id, fatal=False) or {}
 687
 688     def generate_api_headers(
 689             self, *, ytcfg=None, account_syncid=None, session_index=None,
 690             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 691
 692         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 693         headers = {
 694             'X-YouTube-Client-Name': str(
 695                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 696             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 697             'Origin': origin,
 698             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 699             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 700             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 701             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 702         }
 703         if session_index is None:
 704             session_index = self._extract_session_index(ytcfg)
 705         if account_syncid or session_index is not None:
 706             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 707
 708         auth = self._generate_sapisidhash_header(origin)
 709         if auth is not None:
 710             headers['Authorization'] = auth
 711             headers['X-Origin'] = origin
 712         return filter_dict(headers)
 713
 714     def _download_ytcfg(self, client, video_id):
 715         url = {
 716             'web': 'https://www.youtube.com',
 717             'web_music': 'https://music.youtube.com',
 718             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 719         }.get(client)
 720         if not url:
 721             return {}
 722         webpage = self._download_webpage(
 723             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 724         return self.extract_ytcfg(video_id, webpage) or {}
 725
 726     @staticmethod
 727     def _build_api_continuation_query(continuation, ctp=None):
 728         query = {
 729             'continuation': continuation
 730         }
 731         # TODO: Inconsistency with clickTrackingParams.
 732         # Currently we have a fixed ctp contained within context (from ytcfg)
 733         # and a ctp in root query for continuation.
 734         if ctp:
 735             query['clickTracking'] = {'clickTrackingParams': ctp}
 736         return query
 737
 738     @classmethod
 739     def _extract_next_continuation_data(cls, renderer):
 740         next_continuation = try_get(
 741             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 742                        lambda x: x['continuation']['reloadContinuationData']), dict)
 743         if not next_continuation:
 744             return
 745         continuation = next_continuation.get('continuation')
 746         if not continuation:
 747             return
 748         ctp = next_continuation.get('clickTrackingParams')
 749         return cls._build_api_continuation_query(continuation, ctp)
 750
 751     @classmethod
 752     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 753         if isinstance(continuation_ep, dict):
 754             continuation = try_get(
 755                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 756             if not continuation:
 757                 return
 758             ctp = continuation_ep.get('clickTrackingParams')
 759             return cls._build_api_continuation_query(continuation, ctp)
 760
 761     @classmethod
 762     def _extract_continuation(cls, renderer):
 763         next_continuation = cls._extract_next_continuation_data(renderer)
 764         if next_continuation:
 765             return next_continuation
 766
 767         return traverse_obj(renderer, (
 768             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 769             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 770         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 771
 772     @classmethod
 773     def _extract_alerts(cls, data):
 774         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 775             if not isinstance(alert_dict, dict):
 776                 continue
 777             for alert in alert_dict.values():
 778                 alert_type = alert.get('type')
 779                 if not alert_type:
 780                     continue
 781                 message = cls._get_text(alert, 'text')
 782                 if message:
 783                     yield alert_type, message
 784
 785     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 786         errors, warnings = [], []
 787         for alert_type, alert_message in alerts:
 788             if alert_type.lower() == 'error' and fatal:
 789                 errors.append([alert_type, alert_message])
 790             elif alert_message not in self._IGNORED_WARNINGS:
 791                 warnings.append([alert_type, alert_message])
 792
 793         for alert_type, alert_message in (warnings + errors[:-1]):
 794             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 795         if errors:
 796             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 797
 798     def _extract_and_report_alerts(self, data, *args, **kwargs):
 799         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 800
 801     def _extract_badges(self, badge_list: list):
 802         """
 803         Extract known BadgeType's from a list of badge renderers.
 804         @returns [{'type': BadgeType}]
 805         """
 806         icon_type_map = {
 807             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 808             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 809             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC,
 810             'CHECK_CIRCLE_THICK': BadgeType.VERIFIED,
 811             'OFFICIAL_ARTIST_BADGE': BadgeType.VERIFIED,
 812             'CHECK': BadgeType.VERIFIED,
 813         }
 814
 815         badge_style_map = {
 816             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 817             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 818             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW,
 819             'BADGE_STYLE_TYPE_VERIFIED': BadgeType.VERIFIED,
 820             'BADGE_STYLE_TYPE_VERIFIED_ARTIST': BadgeType.VERIFIED,
 821         }
 822
 823         label_map = {
 824             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 825             'private': BadgeType.AVAILABILITY_PRIVATE,
 826             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 827             'live': BadgeType.LIVE_NOW,
 828             'premium': BadgeType.AVAILABILITY_PREMIUM,
 829             'verified': BadgeType.VERIFIED,
 830             'official artist channel': BadgeType.VERIFIED,
 831         }
 832
 833         badges = []
 834         for badge in traverse_obj(badge_list, (..., lambda key, _: re.search(r'[bB]adgeRenderer$', key))):
 835             badge_type = (
 836                 icon_type_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 837                 or badge_style_map.get(traverse_obj(badge, 'style'))
 838             )
 839             if badge_type:
 840                 badges.append({'type': badge_type})
 841                 continue
 842
 843             # fallback, won't work in some languages
 844             label = traverse_obj(
 845                 badge, 'label', ('accessibilityData', 'label'), 'tooltip', 'iconTooltip', get_all=False, expected_type=str, default='')
 846             for match, label_badge_type in label_map.items():
 847                 if match in label.lower():
 848                     badges.append({'type': label_badge_type})
 849                     break
 850
 851         return badges
 852
 853     @staticmethod
 854     def _has_badge(badges, badge_type):
 855         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 856
 857     @staticmethod
 858     def _get_text(data, *path_list, max_runs=None):
 859         for path in path_list or [None]:
 860             if path is None:
 861                 obj = [data]
 862             else:
 863                 obj = traverse_obj(data, path, default=[])
 864                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 865                     obj = [obj]
 866             for item in obj:
 867                 text = try_get(item, lambda x: x['simpleText'], str)
 868                 if text:
 869                     return text
 870                 runs = try_get(item, lambda x: x['runs'], list) or []
 871                 if not runs and isinstance(item, list):
 872                     runs = item
 873
 874                 runs = runs[:min(len(runs), max_runs or len(runs))]
 875                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
 876                 if text:
 877                     return text
 878
 879     def _get_count(self, data, *path_list):
 880         count_text = self._get_text(data, *path_list) or ''
 881         count = parse_count(count_text)
 882         if count is None:
 883             count = str_to_int(
 884                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 885         return count
 886
 887     @staticmethod
 888     def _extract_thumbnails(data, *path_list):
 889         """
 890         Extract thumbnails from thumbnails dict
 891         @param path_list: path list to level that contains 'thumbnails' key
 892         """
 893         thumbnails = []
 894         for path in path_list or [()]:
 895             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...)):
 896                 thumbnail_url = url_or_none(thumbnail.get('url'))
 897                 if not thumbnail_url:
 898                     continue
 899                 # Sometimes youtube gives a wrong thumbnail URL. See:
 900                 # https://github.com/yt-dlp/yt-dlp/issues/233
 901                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 902                 if 'maxresdefault' in thumbnail_url:
 903                     thumbnail_url = thumbnail_url.split('?')[0]
 904                 thumbnails.append({
 905                     'url': thumbnail_url,
 906                     'height': int_or_none(thumbnail.get('height')),
 907                     'width': int_or_none(thumbnail.get('width')),
 908                 })
 909         return thumbnails
 910
 911     @staticmethod
 912     def extract_relative_time(relative_time_text):
 913         """
 914         Extracts a relative time from string and converts to dt object
 915         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today', '8 yr ago'
 916         """
 917
 918         # XXX: this could be moved to a general function in utils/_utils.py
 919         # The relative time text strings are roughly the same as what
 920         # Javascript's Intl.RelativeTimeFormat function generates.
 921         # See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/RelativeTimeFormat
 922         mobj = re.search(
 923             r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>sec(?:ond)?|s|min(?:ute)?|h(?:our|r)?|d(?:ay)?|w(?:eek|k)?|mo(?:nth)?|y(?:ear|r)?)s?\s*ago',
 924             relative_time_text)
 925         if mobj:
 926             start = mobj.group('start')
 927             if start:
 928                 return datetime_from_str(start)
 929             try:
 930                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 931             except ValueError:
 932                 return None
 933
 934     def _parse_time_text(self, text):
 935         if not text:
 936             return
 937         dt_ = self.extract_relative_time(text)
 938         timestamp = None
 939         if isinstance(dt_, dt.datetime):
 940             timestamp = calendar.timegm(dt_.timetuple())
 941
 942         if timestamp is None:
 943             timestamp = (
 944                 unified_timestamp(text) or unified_timestamp(
 945                     self._search_regex(
 946                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 947                         text.lower(), 'time text', default=None)))
 948
 949         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 950             self.report_warning(
 951                 f'Cannot parse localized time text "{text}"', only_once=True)
 952         return timestamp
 953
 954     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 955                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 956                           default_client='web'):
 957         raise_for_incomplete = bool(self._configuration_arg('raise_incomplete_data', ie_key=YoutubeIE))
 958         # Incomplete Data should be a warning by default when retries are exhausted, while other errors should be fatal.
 959         icd_retries = iter(self.RetryManager(fatal=raise_for_incomplete))
 960         icd_rm = next(icd_retries)
 961         main_retries = iter(self.RetryManager())
 962         main_rm = next(main_retries)
 963         # Manual retry loop for multiple RetryManagers
 964         # The proper RetryManager MUST be advanced after an error
 965         # and its result MUST be checked if the manager is non fatal
 966         while True:
 967             try:
 968                 response = self._call_api(
 969                     ep=ep, fatal=True, headers=headers,
 970                     video_id=item_id, query=query, note=note,
 971                     context=self._extract_context(ytcfg, default_client),
 972                     api_key=self._extract_api_key(ytcfg, default_client),
 973                     api_hostname=api_hostname, default_client=default_client)
 974             except ExtractorError as e:
 975                 if not isinstance(e.cause, network_exceptions):
 976                     return self._error_or_warning(e, fatal=fatal)
 977                 elif not isinstance(e.cause, HTTPError):
 978                     main_rm.error = e
 979                     next(main_retries)
 980                     continue
 981
 982                 first_bytes = e.cause.response.read(512)
 983                 if not is_html(first_bytes):
 984                     yt_error = try_get(
 985                         self._parse_json(
 986                             self._webpage_read_content(e.cause.response, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 987                         lambda x: x['error']['message'], str)
 988                     if yt_error:
 989                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 990                 # Downloading page may result in intermittent 5xx HTTP error
 991                 # Sometimes a 404 is also received. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 992                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 993                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 994                 if e.cause.status not in (403, 429):
 995                     main_rm.error = e
 996                     next(main_retries)
 997                     continue
 998                 return self._error_or_warning(e, fatal=fatal)
 999
1000             try:
1001                 self._extract_and_report_alerts(response, only_once=True)
1002             except ExtractorError as e:
1003                 # YouTube's servers may return errors we want to retry on in a 200 OK response
1004                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
1005                 if 'unknown error' in e.msg.lower():
1006                     main_rm.error = e
1007                     next(main_retries)
1008                     continue
1009                 return self._error_or_warning(e, fatal=fatal)
1010             # Youtube sometimes sends incomplete data
1011             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
1012             if not traverse_obj(response, *variadic(check_get_keys)):
1013                 icd_rm.error = ExtractorError('Incomplete data received', expected=True)
1014                 should_retry = next(icd_retries, None)
1015                 if not should_retry:
1016                     return None
1017                 continue
1018
1019             return response
1020
1021     @staticmethod
1022     def is_music_url(url):
1023         return re.match(r'(https?://)?music\.youtube\.com/', url) is not None
1024
1025     def _extract_video(self, renderer):
1026         video_id = renderer.get('videoId')
1027
1028         reel_header_renderer = traverse_obj(renderer, (
1029             'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
1030             'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
1031
1032         title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
1033         description = self._get_text(renderer, 'descriptionSnippet')
1034
1035         duration = int_or_none(renderer.get('lengthSeconds'))
1036         if duration is None:
1037             duration = parse_duration(self._get_text(
1038                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
1039         if duration is None:
1040             # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
1041             duration = parse_duration(self._search_regex(
1042                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
1043                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
1044                 video_id, default=None, group='duration'))
1045
1046         channel_id = traverse_obj(
1047             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
1048             expected_type=str, get_all=False)
1049         if not channel_id:
1050             channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
1051
1052         channel_id = self.ucid_or_none(channel_id)
1053
1054         overlay_style = traverse_obj(
1055             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
1056             get_all=False, expected_type=str)
1057         badges = self._extract_badges(traverse_obj(renderer, 'badges'))
1058         owner_badges = self._extract_badges(traverse_obj(renderer, 'ownerBadges'))
1059         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
1060             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
1061             expected_type=str)) or ''
1062         url = f'https://www.youtube.com/watch?v={video_id}'
1063         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
1064             url = f'https://www.youtube.com/shorts/{video_id}'
1065
1066         time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
1067                      or self._get_text(reel_header_renderer, 'timestampText') or '')
1068         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
1069
1070         live_status = (
1071             'is_upcoming' if scheduled_timestamp is not None
1072             else 'was_live' if 'streamed' in time_text.lower()
1073             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
1074             else None)
1075
1076         # videoInfo is a string like '50K views • 10 years ago'.
1077         view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
1078         view_count = (0 if 'no views' in view_count_text.lower()
1079                       else self._get_count({'simpleText': view_count_text}))
1080         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
1081
1082         channel = (self._get_text(renderer, 'ownerText', 'shortBylineText')
1083                    or self._get_text(reel_header_renderer, 'channelTitleText'))
1084
1085         channel_handle = traverse_obj(renderer, (
1086             'shortBylineText', 'runs', ..., 'navigationEndpoint',
1087             (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl'))),
1088             expected_type=self.handle_from_url, get_all=False)
1089         return {
1090             '_type': 'url',
1091             'ie_key': YoutubeIE.ie_key(),
1092             'id': video_id,
1093             'url': url,
1094             'title': title,
1095             'description': description,
1096             'duration': duration,
1097             'channel_id': channel_id,
1098             'channel': channel,
1099             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
1100             'uploader': channel,
1101             'uploader_id': channel_handle,
1102             'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
1103             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
1104             'timestamp': (self._parse_time_text(time_text)
1105                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
1106                           else None),
1107             'release_timestamp': scheduled_timestamp,
1108             'availability':
1109                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
1110                 else self._availability(
1111                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
1112                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
1113                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
1114                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
1115             view_count_field: view_count,
1116             'live_status': live_status,
1117             'channel_is_verified': True if self._has_badge(owner_badges, BadgeType.VERIFIED) else None
1118         }
1119
1120
1121 class YoutubeIE(YoutubeBaseInfoExtractor):
1122     IE_DESC = 'YouTube'
1123     _VALID_URL = r"""(?x)^
1124                      (
1125                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
1126                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
1127                             (?:www\.)?deturl\.com/www\.youtube\.com|
1128                             (?:www\.)?pwnyoutube\.com|
1129                             (?:www\.)?hooktube\.com|
1130                             (?:www\.)?yourepeat\.com|
1131                             tube\.majestyc\.net|
1132                             %(invidious)s|
1133                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
1134                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
1135                          (?:                                                  # the various things that can precede the ID:
1136                              (?:(?:v|embed|e|shorts|live)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
1137                              |(?:                                             # or the v= param in all its forms
1138                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
1139                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
1140                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
1141                                  v=
1142                              )
1143                          ))
1144                          |(?:
1145                             youtu\.be|                                        # just youtu.be/xxxx
1146                             vid\.plus|                                        # or vid.plus/xxxx
1147                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
1148                             %(invidious)s
1149                          )/
1150                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1151                          )
1152                      )?                                                       # all until now is optional -> you can pass the naked ID
1153                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1154                      (?(1).+)?                                                # if we found the ID, everything can follow
1155                      (?:\#|$)""" % {
1156         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1157     }
1158     _EMBED_REGEX = [
1159         r'''(?x)
1160             (?:
1161                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1162                 data-video-url=|
1163                 <embed[^>]+?src=|
1164                 embedSWF\(?:\s*|
1165                 <object[^>]+data=|
1166                 new\s+SWFObject\(
1167             )
1168             (["\'])
1169                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1170                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1171             \1''',
1172         # https://wordpress.org/plugins/lazy-load-for-videos/
1173         r'''(?xs)
1174             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1175             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1176     ]
1177     _RETURN_TYPE = 'video'  # XXX: How to handle multifeed?
1178
1179     _PLAYER_INFO_RE = (
1180         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1181         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1182         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1183     )
1184     _formats = {  # NB: Used in YoutubeWebArchiveIE and GoogleDriveIE
1185         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1186         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1187         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1188         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1189         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1190         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1191         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1192         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1193         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1194         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1195         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1196         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1197         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1198         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1199         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1200         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1201         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1202         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1203
1204
1205         # 3D videos
1206         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1207         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1208         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1209         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1210         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1211         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1212         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1213
1214         # Apple HTTP Live Streaming
1215         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1216         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1217         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1218         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1219         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1220         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1221         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1222         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1223
1224         # DASH mp4 video
1225         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1226         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1227         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1228         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1229         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1230         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1231         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1232         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1233         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1234         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1235         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1236         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1237
1238         # Dash mp4 audio
1239         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1240         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1241         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1242         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1243         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1244         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1245         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1246
1247         # Dash webm
1248         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1249         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1250         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1251         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1252         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1253         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1254         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1255         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1256         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1257         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1258         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1259         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1260         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1261         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1262         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1263         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1264         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1265         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1266         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1267         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1268         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1269         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1270
1271         # Dash webm audio
1272         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1273         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1274
1275         # Dash webm audio with opus inside
1276         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1277         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1278         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1279
1280         # RTMP (unnamed)
1281         '_rtmp': {'protocol': 'rtmp'},
1282
1283         # av01 video only formats sometimes served with "unknown" codecs
1284         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1285         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1286         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1287         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1288         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1289         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1290         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1291         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1292     }
1293     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1294
1295     _GEO_BYPASS = False
1296
1297     IE_NAME = 'youtube'
1298     _TESTS = [
1299         {
1300             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1301             'info_dict': {
1302                 'id': 'BaW_jenozKc',
1303                 'ext': 'mp4',
1304                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1305                 'channel': 'Philipp Hagemeister',
1306                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1307                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1308                 'upload_date': '20121002',
1309                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1310                 'categories': ['Science & Technology'],
1311                 'tags': ['youtube-dl'],
1312                 'duration': 10,
1313                 'view_count': int,
1314                 'like_count': int,
1315                 'availability': 'public',
1316                 'playable_in_embed': True,
1317                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1318                 'live_status': 'not_live',
1319                 'age_limit': 0,
1320                 'start_time': 1,
1321                 'end_time': 9,
1322                 'comment_count': int,
1323                 'channel_follower_count': int,
1324                 'uploader': 'Philipp Hagemeister',
1325                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
1326                 'uploader_id': '@PhilippHagemeister',
1327                 'heatmap': 'count:100',
1328             }
1329         },
1330         {
1331             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1332             'note': 'Embed-only video (#1746)',
1333             'info_dict': {
1334                 'id': 'yZIXLfi8CZQ',
1335                 'ext': 'mp4',
1336                 'upload_date': '20120608',
1337                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1338                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1339                 'age_limit': 18,
1340             },
1341             'skip': 'Private video',
1342         },
1343         {
1344             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1345             'note': 'Use the first video ID in the URL',
1346             'info_dict': {
1347                 'id': 'BaW_jenozKc',
1348                 'ext': 'mp4',
1349                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1350                 'channel': 'Philipp Hagemeister',
1351                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1352                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1353                 'upload_date': '20121002',
1354                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1355                 'categories': ['Science & Technology'],
1356                 'tags': ['youtube-dl'],
1357                 'duration': 10,
1358                 'view_count': int,
1359                 'like_count': int,
1360                 'availability': 'public',
1361                 'playable_in_embed': True,
1362                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1363                 'live_status': 'not_live',
1364                 'age_limit': 0,
1365                 'comment_count': int,
1366                 'channel_follower_count': int,
1367                 'uploader': 'Philipp Hagemeister',
1368                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
1369                 'uploader_id': '@PhilippHagemeister',
1370                 'heatmap': 'count:100',
1371             },
1372             'params': {
1373                 'skip_download': True,
1374             },
1375         },
1376         {
1377             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1378             'note': '256k DASH audio (format 141) via DASH manifest',
1379             'info_dict': {
1380                 'id': 'a9LDPn-MO4I',
1381                 'ext': 'm4a',
1382                 'upload_date': '20121002',
1383                 'description': '',
1384                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1385             },
1386             'params': {
1387                 'youtube_include_dash_manifest': True,
1388                 'format': '141',
1389             },
1390             'skip': 'format 141 not served anymore',
1391         },
1392         # DASH manifest with encrypted signature
1393         {
1394             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1395             'info_dict': {
1396                 'id': 'IB3lcPjvWLA',
1397                 'ext': 'm4a',
1398                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1399                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1400                 'duration': 244,
1401                 'upload_date': '20131011',
1402                 'abr': 129.495,
1403                 'like_count': int,
1404                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1405                 'playable_in_embed': True,
1406                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1407                 'view_count': int,
1408                 'track': 'The Spark',
1409                 'live_status': 'not_live',
1410                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1411                 'channel': 'Afrojack',
1412                 'tags': 'count:19',
1413                 'availability': 'public',
1414                 'categories': ['Music'],
1415                 'age_limit': 0,
1416                 'alt_title': 'The Spark',
1417                 'channel_follower_count': int,
1418                 'uploader': 'Afrojack',
1419                 'uploader_url': 'https://www.youtube.com/@Afrojack',
1420                 'uploader_id': '@Afrojack',
1421             },
1422             'params': {
1423                 'youtube_include_dash_manifest': True,
1424                 'format': '141/bestaudio[ext=m4a]',
1425             },
1426         },
1427         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1428         {
1429             'note': 'Embed allowed age-gate video',
1430             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1431             'info_dict': {
1432                 'id': 'HtVdAasjOgU',
1433                 'ext': 'mp4',
1434                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1435                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1436                 'duration': 142,
1437                 'upload_date': '20140605',
1438                 'age_limit': 18,
1439                 'categories': ['Gaming'],
1440                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1441                 'availability': 'needs_auth',
1442                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1443                 'like_count': int,
1444                 'channel': 'The Witcher',
1445                 'live_status': 'not_live',
1446                 'tags': 'count:17',
1447                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1448                 'playable_in_embed': True,
1449                 'view_count': int,
1450                 'channel_follower_count': int,
1451                 'uploader': 'The Witcher',
1452                 'uploader_url': 'https://www.youtube.com/@thewitcher',
1453                 'uploader_id': '@thewitcher',
1454                 'comment_count': int,
1455                 'channel_is_verified': True,
1456                 'heatmap': 'count:100',
1457             },
1458         },
1459         {
1460             'note': 'Age-gate video with embed allowed in public site',
1461             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1462             'info_dict': {
1463                 'id': 'HsUATh_Nc2U',
1464                 'ext': 'mp4',
1465                 'title': 'Godzilla 2 (Official Video)',
1466                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1467                 'upload_date': '20200408',
1468                 'age_limit': 18,
1469                 'availability': 'needs_auth',
1470                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1471                 'channel': 'FlyingKitty',
1472                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1473                 'view_count': int,
1474                 'categories': ['Entertainment'],
1475                 'live_status': 'not_live',
1476                 'tags': ['Flyingkitty', 'godzilla 2'],
1477                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1478                 'like_count': int,
1479                 'duration': 177,
1480                 'playable_in_embed': True,
1481                 'channel_follower_count': int,
1482                 'uploader': 'FlyingKitty',
1483                 'uploader_url': 'https://www.youtube.com/@FlyingKitty900',
1484                 'uploader_id': '@FlyingKitty900',
1485                 'comment_count': int,
1486                 'channel_is_verified': True,
1487             },
1488         },
1489         {
1490             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1491             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1492             'info_dict': {
1493                 'id': 'Tq92D6wQ1mg',
1494                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1495                 'ext': 'mp4',
1496                 'upload_date': '20191228',
1497                 'description': 'md5:17eccca93a786d51bc67646756894066',
1498                 'age_limit': 18,
1499                 'like_count': int,
1500                 'availability': 'needs_auth',
1501                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1502                 'view_count': int,
1503                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1504                 'channel': 'Projekt Melody',
1505                 'live_status': 'not_live',
1506                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1507                 'playable_in_embed': True,
1508                 'categories': ['Entertainment'],
1509                 'duration': 106,
1510                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1511                 'comment_count': int,
1512                 'channel_follower_count': int,
1513                 'uploader': 'Projekt Melody',
1514                 'uploader_url': 'https://www.youtube.com/@ProjektMelody',
1515                 'uploader_id': '@ProjektMelody',
1516             },
1517         },
1518         {
1519             'note': 'Non-Agegated non-embeddable video',
1520             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1521             'info_dict': {
1522                 'id': 'MeJVWBSsPAY',
1523                 'ext': 'mp4',
1524                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1525                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1526                 'upload_date': '20130730',
1527                 'track': 'Such mich find mich',
1528                 'age_limit': 0,
1529                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1530                 'like_count': int,
1531                 'playable_in_embed': False,
1532                 'creator': 'OOMPH!',
1533                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1534                 'view_count': int,
1535                 'alt_title': 'Such mich find mich',
1536                 'duration': 210,
1537                 'channel': 'Herr Lurik',
1538                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1539                 'categories': ['Music'],
1540                 'availability': 'public',
1541                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1542                 'live_status': 'not_live',
1543                 'artist': 'OOMPH!',
1544                 'channel_follower_count': int,
1545                 'uploader': 'Herr Lurik',
1546                 'uploader_url': 'https://www.youtube.com/@HerrLurik',
1547                 'uploader_id': '@HerrLurik',
1548             },
1549         },
1550         {
1551             'note': 'Non-bypassable age-gated video',
1552             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1553             'only_matching': True,
1554         },
1555         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1556         # YouTube Red ad is not captured for creator
1557         {
1558             'url': '__2ABJjxzNo',
1559             'info_dict': {
1560                 'id': '__2ABJjxzNo',
1561                 'ext': 'mp4',
1562                 'duration': 266,
1563                 'upload_date': '20100430',
1564                 'creator': 'deadmau5',
1565                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1566                 'title': 'Deadmau5 - Some Chords (HD)',
1567                 'alt_title': 'Some Chords',
1568                 'availability': 'public',
1569                 'tags': 'count:14',
1570                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1571                 'view_count': int,
1572                 'live_status': 'not_live',
1573                 'channel': 'deadmau5',
1574                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1575                 'like_count': int,
1576                 'track': 'Some Chords',
1577                 'artist': 'deadmau5',
1578                 'playable_in_embed': True,
1579                 'age_limit': 0,
1580                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1581                 'categories': ['Music'],
1582                 'album': 'Some Chords',
1583                 'channel_follower_count': int,
1584                 'uploader': 'deadmau5',
1585                 'uploader_url': 'https://www.youtube.com/@deadmau5',
1586                 'uploader_id': '@deadmau5',
1587             },
1588             'expected_warnings': [
1589                 'DASH manifest missing',
1590             ]
1591         },
1592         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1593         {
1594             'url': 'lqQg6PlCWgI',
1595             'info_dict': {
1596                 'id': 'lqQg6PlCWgI',
1597                 'ext': 'mp4',
1598                 'duration': 6085,
1599                 'upload_date': '20150827',
1600                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1601                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1602                 'like_count': int,
1603                 'release_timestamp': 1343767800,
1604                 'playable_in_embed': True,
1605                 'categories': ['Sports'],
1606                 'release_date': '20120731',
1607                 'channel': 'Olympics',
1608                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1609                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1610                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1611                 'age_limit': 0,
1612                 'availability': 'public',
1613                 'live_status': 'was_live',
1614                 'view_count': int,
1615                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1616                 'channel_follower_count': int,
1617                 'uploader': 'Olympics',
1618                 'uploader_url': 'https://www.youtube.com/@Olympics',
1619                 'uploader_id': '@Olympics',
1620                 'channel_is_verified': True,
1621             },
1622             'params': {
1623                 'skip_download': 'requires avconv',
1624             }
1625         },
1626         # Non-square pixels
1627         {
1628             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1629             'info_dict': {
1630                 'id': '_b-2C3KPAM0',
1631                 'ext': 'mp4',
1632                 'stretched_ratio': 16 / 9.,
1633                 'duration': 85,
1634                 'upload_date': '20110310',
1635                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1636                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1637                 'playable_in_embed': True,
1638                 'channel': '孫ᄋᄅ',
1639                 'age_limit': 0,
1640                 'tags': 'count:11',
1641                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1642                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1643                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1644                 'view_count': int,
1645                 'categories': ['People & Blogs'],
1646                 'like_count': int,
1647                 'live_status': 'not_live',
1648                 'availability': 'unlisted',
1649                 'comment_count': int,
1650                 'channel_follower_count': int,
1651                 'uploader': '孫ᄋᄅ',
1652                 'uploader_url': 'https://www.youtube.com/@AllenMeow',
1653                 'uploader_id': '@AllenMeow',
1654             },
1655         },
1656         # url_encoded_fmt_stream_map is empty string
1657         {
1658             'url': 'qEJwOuvDf7I',
1659             'info_dict': {
1660                 'id': 'qEJwOuvDf7I',
1661                 'ext': 'webm',
1662                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1663                 'description': '',
1664                 'upload_date': '20150404',
1665             },
1666             'params': {
1667                 'skip_download': 'requires avconv',
1668             },
1669             'skip': 'This live event has ended.',
1670         },
1671         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1672         {
1673             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1674             'info_dict': {
1675                 'id': 'FIl7x6_3R5Y',
1676                 'ext': 'webm',
1677                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1678                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1679                 'duration': 220,
1680                 'upload_date': '20150625',
1681                 'formats': 'mincount:31',
1682             },
1683             'skip': 'not actual anymore',
1684         },
1685         # DASH manifest with segment_list
1686         {
1687             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1688             'md5': '8ce563a1d667b599d21064e982ab9e31',
1689             'info_dict': {
1690                 'id': 'CsmdDsKjzN8',
1691                 'ext': 'mp4',
1692                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1693                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1694                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1695             },
1696             'params': {
1697                 'youtube_include_dash_manifest': True,
1698                 'format': '135',  # bestvideo
1699             },
1700             'skip': 'This live event has ended.',
1701         },
1702         {
1703             # Multifeed videos (multiple cameras), URL can be of any Camera
1704             # TODO: fix multifeed titles
1705             'url': 'https://www.youtube.com/watch?v=zaPI8MvL8pg',
1706             'info_dict': {
1707                 'id': 'zaPI8MvL8pg',
1708                 'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04',
1709                 'description': 'md5:563ccbc698b39298481ca3c571169519',
1710             },
1711             'playlist': [{
1712                 'info_dict': {
1713                     'id': 'j5yGuxZ8lLU',
1714                     'ext': 'mp4',
1715                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Chris)',
1716                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1717                     'duration': 10120,
1718                     'channel_follower_count': int,
1719                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1720                     'availability': 'public',
1721                     'playable_in_embed': True,
1722                     'upload_date': '20131105',
1723                     'categories': ['Gaming'],
1724                     'live_status': 'was_live',
1725                     'tags': 'count:24',
1726                     'release_timestamp': 1383701910,
1727                     'thumbnail': 'https://i.ytimg.com/vi/j5yGuxZ8lLU/maxresdefault.jpg',
1728                     'comment_count': int,
1729                     'age_limit': 0,
1730                     'like_count': int,
1731                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1732                     'channel': 'WiiLikeToPlay',
1733                     'view_count': int,
1734                     'release_date': '20131106',
1735                     'uploader': 'WiiLikeToPlay',
1736                     'uploader_id': '@WLTP',
1737                     'uploader_url': 'https://www.youtube.com/@WLTP',
1738                 },
1739             }, {
1740                 'info_dict': {
1741                     'id': 'zaPI8MvL8pg',
1742                     'ext': 'mp4',
1743                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Tyson)',
1744                     'availability': 'public',
1745                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1746                     'channel': 'WiiLikeToPlay',
1747                     'channel_follower_count': int,
1748                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1749                     'duration': 10108,
1750                     'age_limit': 0,
1751                     'like_count': int,
1752                     'tags': 'count:24',
1753                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1754                     'release_timestamp': 1383701915,
1755                     'comment_count': int,
1756                     'upload_date': '20131105',
1757                     'thumbnail': 'https://i.ytimg.com/vi/zaPI8MvL8pg/maxresdefault.jpg',
1758                     'release_date': '20131106',
1759                     'playable_in_embed': True,
1760                     'live_status': 'was_live',
1761                     'categories': ['Gaming'],
1762                     'view_count': int,
1763                     'uploader': 'WiiLikeToPlay',
1764                     'uploader_id': '@WLTP',
1765                     'uploader_url': 'https://www.youtube.com/@WLTP',
1766                 },
1767             }, {
1768                 'info_dict': {
1769                     'id': 'R7r3vfO7Hao',
1770                     'ext': 'mp4',
1771                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Spencer)',
1772                     'thumbnail': 'https://i.ytimg.com/vi/R7r3vfO7Hao/maxresdefault.jpg',
1773                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1774                     'like_count': int,
1775                     'availability': 'public',
1776                     'playable_in_embed': True,
1777                     'upload_date': '20131105',
1778                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1779                     'channel_follower_count': int,
1780                     'tags': 'count:24',
1781                     'release_date': '20131106',
1782                     'comment_count': int,
1783                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1784                     'channel': 'WiiLikeToPlay',
1785                     'categories': ['Gaming'],
1786                     'release_timestamp': 1383701914,
1787                     'live_status': 'was_live',
1788                     'age_limit': 0,
1789                     'duration': 10128,
1790                     'view_count': int,
1791                     'uploader': 'WiiLikeToPlay',
1792                     'uploader_id': '@WLTP',
1793                     'uploader_url': 'https://www.youtube.com/@WLTP',
1794                 },
1795             }],
1796             'params': {'skip_download': True},
1797         },
1798         {
1799             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1800             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1801             'info_dict': {
1802                 'id': 'gVfLd0zydlo',
1803                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1804             },
1805             'playlist_count': 2,
1806             'skip': 'Not multifeed anymore',
1807         },
1808         {
1809             'url': 'https://vid.plus/FlRa-iH7PGw',
1810             'only_matching': True,
1811         },
1812         {
1813             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1814             'only_matching': True,
1815         },
1816         {
1817             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1818             # Also tests cut-off URL expansion in video description (see
1819             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1820             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1821             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1822             'info_dict': {
1823                 'id': 'lsguqyKfVQg',
1824                 'ext': 'mp4',
1825                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1826                 'alt_title': 'Dark Walk',
1827                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1828                 'duration': 133,
1829                 'upload_date': '20151119',
1830                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1831                 'track': 'Dark Walk',
1832                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1833                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1834                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1835                 'categories': ['Film & Animation'],
1836                 'view_count': int,
1837                 'live_status': 'not_live',
1838                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1839                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1840                 'tags': 'count:13',
1841                 'availability': 'public',
1842                 'channel': 'IronSoulElf',
1843                 'playable_in_embed': True,
1844                 'like_count': int,
1845                 'age_limit': 0,
1846                 'channel_follower_count': int
1847             },
1848             'params': {
1849                 'skip_download': True,
1850             },
1851         },
1852         {
1853             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1854             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1855             'only_matching': True,
1856         },
1857         {
1858             # Video with yt:stretch=17:0
1859             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1860             'info_dict': {
1861                 'id': 'Q39EVAstoRM',
1862                 'ext': 'mp4',
1863                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1864                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1865                 'upload_date': '20151107',
1866             },
1867             'params': {
1868                 'skip_download': True,
1869             },
1870             'skip': 'This video does not exist.',
1871         },
1872         {
1873             # Video with incomplete 'yt:stretch=16:'
1874             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1875             'only_matching': True,
1876         },
1877         {
1878             # Video licensed under Creative Commons
1879             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1880             'info_dict': {
1881                 'id': 'M4gD1WSo5mA',
1882                 'ext': 'mp4',
1883                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1884                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1885                 'duration': 721,
1886                 'upload_date': '20150128',
1887                 'license': 'Creative Commons Attribution license (reuse allowed)',
1888                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1889                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1890                 'like_count': int,
1891                 'age_limit': 0,
1892                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1893                 'channel': 'The Berkman Klein Center for Internet & Society',
1894                 'availability': 'public',
1895                 'view_count': int,
1896                 'categories': ['Education'],
1897                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1898                 'live_status': 'not_live',
1899                 'playable_in_embed': True,
1900                 'channel_follower_count': int,
1901                 'chapters': list,
1902                 'uploader': 'The Berkman Klein Center for Internet & Society',
1903                 'uploader_id': '@BKCHarvard',
1904                 'uploader_url': 'https://www.youtube.com/@BKCHarvard',
1905             },
1906             'params': {
1907                 'skip_download': True,
1908             },
1909         },
1910         {
1911             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1912             'info_dict': {
1913                 'id': 'eQcmzGIKrzg',
1914                 'ext': 'mp4',
1915                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1916                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1917                 'duration': 4060,
1918                 'upload_date': '20151120',
1919                 'license': 'Creative Commons Attribution license (reuse allowed)',
1920                 'playable_in_embed': True,
1921                 'tags': 'count:12',
1922                 'like_count': int,
1923                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1924                 'age_limit': 0,
1925                 'availability': 'public',
1926                 'categories': ['News & Politics'],
1927                 'channel': 'Bernie Sanders',
1928                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1929                 'view_count': int,
1930                 'live_status': 'not_live',
1931                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1932                 'comment_count': int,
1933                 'channel_follower_count': int,
1934                 'chapters': list,
1935                 'uploader': 'Bernie Sanders',
1936                 'uploader_url': 'https://www.youtube.com/@BernieSanders',
1937                 'uploader_id': '@BernieSanders',
1938                 'channel_is_verified': True,
1939                 'heatmap': 'count:100',
1940             },
1941             'params': {
1942                 'skip_download': True,
1943             },
1944         },
1945         {
1946             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1947             'only_matching': True,
1948         },
1949         {
1950             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1951             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1952             'only_matching': True,
1953         },
1954         {
1955             # Rental video preview
1956             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1957             'info_dict': {
1958                 'id': 'uGpuVWrhIzE',
1959                 'ext': 'mp4',
1960                 'title': 'Piku - Trailer',
1961                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1962                 'upload_date': '20150811',
1963                 'license': 'Standard YouTube License',
1964             },
1965             'params': {
1966                 'skip_download': True,
1967             },
1968             'skip': 'This video is not available.',
1969         },
1970         {
1971             # YouTube Red video with episode data
1972             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1973             'info_dict': {
1974                 'id': 'iqKdEhx-dD4',
1975                 'ext': 'mp4',
1976                 'title': 'Isolation - Mind Field (Ep 1)',
1977                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1978                 'duration': 2085,
1979                 'upload_date': '20170118',
1980                 'series': 'Mind Field',
1981                 'season_number': 1,
1982                 'episode_number': 1,
1983                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1984                 'tags': 'count:12',
1985                 'view_count': int,
1986                 'availability': 'public',
1987                 'age_limit': 0,
1988                 'channel': 'Vsauce',
1989                 'episode': 'Episode 1',
1990                 'categories': ['Entertainment'],
1991                 'season': 'Season 1',
1992                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1993                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1994                 'like_count': int,
1995                 'playable_in_embed': True,
1996                 'live_status': 'not_live',
1997                 'channel_follower_count': int,
1998                 'uploader': 'Vsauce',
1999                 'uploader_url': 'https://www.youtube.com/@Vsauce',
2000                 'uploader_id': '@Vsauce',
2001                 'comment_count': int,
2002                 'channel_is_verified': True,
2003             },
2004             'params': {
2005                 'skip_download': True,
2006             },
2007             'expected_warnings': [
2008                 'Skipping DASH manifest',
2009             ],
2010         },
2011         {
2012             # The following content has been identified by the YouTube community
2013             # as inappropriate or offensive to some audiences.
2014             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
2015             'info_dict': {
2016                 'id': '6SJNVb0GnPI',
2017                 'ext': 'mp4',
2018                 'title': 'Race Differences in Intelligence',
2019                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
2020                 'duration': 965,
2021                 'upload_date': '20140124',
2022             },
2023             'params': {
2024                 'skip_download': True,
2025             },
2026             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
2027         },
2028         {
2029             # itag 212
2030             'url': '1t24XAntNCY',
2031             'only_matching': True,
2032         },
2033         {
2034             # geo restricted to JP
2035             'url': 'sJL6WA-aGkQ',
2036             'only_matching': True,
2037         },
2038         {
2039             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
2040             'only_matching': True,
2041         },
2042         {
2043             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
2044             'only_matching': True,
2045         },
2046         {
2047             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
2048             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
2049             'only_matching': True,
2050         },
2051         {
2052             # DRM protected
2053             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
2054             'only_matching': True,
2055         },
2056         {
2057             # Video with unsupported adaptive stream type formats
2058             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
2059             'info_dict': {
2060                 'id': 'Z4Vy8R84T1U',
2061                 'ext': 'mp4',
2062                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
2063                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
2064                 'duration': 433,
2065                 'upload_date': '20130923',
2066                 'formats': 'maxcount:10',
2067             },
2068             'params': {
2069                 'skip_download': True,
2070                 'youtube_include_dash_manifest': False,
2071             },
2072             'skip': 'not actual anymore',
2073         },
2074         {
2075             # Youtube Music Auto-generated description
2076             # TODO: fix metadata extraction
2077             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2078             'info_dict': {
2079                 'id': 'MgNrAu2pzNs',
2080                 'ext': 'mp4',
2081                 'title': 'Voyeur Girl',
2082                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
2083                 'upload_date': '20190312',
2084                 'artists': ['Stephen'],
2085                 'creators': ['Stephen'],
2086                 'track': 'Voyeur Girl',
2087                 'album': 'it\'s too much love to know my dear',
2088                 'release_date': '20190313',
2089                 'alt_title': 'Voyeur Girl',
2090                 'view_count': int,
2091                 'playable_in_embed': True,
2092                 'like_count': int,
2093                 'categories': ['Music'],
2094                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
2095                 'channel': 'Stephen',  # TODO: should be "Stephen - Topic"
2096                 'uploader': 'Stephen',
2097                 'availability': 'public',
2098                 'duration': 169,
2099                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
2100                 'age_limit': 0,
2101                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
2102                 'tags': 'count:11',
2103                 'live_status': 'not_live',
2104                 'channel_follower_count': int
2105             },
2106             'params': {
2107                 'skip_download': True,
2108             },
2109         },
2110         {
2111             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
2112             'only_matching': True,
2113         },
2114         {
2115             # invalid -> valid video id redirection
2116             'url': 'DJztXj2GPfl',
2117             'info_dict': {
2118                 'id': 'DJztXj2GPfk',
2119                 'ext': 'mp4',
2120                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
2121                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
2122                 'upload_date': '20090125',
2123                 'artist': 'Panjabi MC',
2124                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
2125                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
2126             },
2127             'params': {
2128                 'skip_download': True,
2129             },
2130             'skip': 'Video unavailable',
2131         },
2132         {
2133             # empty description results in an empty string
2134             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
2135             'info_dict': {
2136                 'id': 'x41yOUIvK2k',
2137                 'ext': 'mp4',
2138                 'title': 'IMG 3456',
2139                 'description': '',
2140                 'upload_date': '20170613',
2141                 'view_count': int,
2142                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
2143                 'like_count': int,
2144                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
2145                 'tags': [],
2146                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
2147                 'availability': 'public',
2148                 'age_limit': 0,
2149                 'categories': ['Pets & Animals'],
2150                 'duration': 7,
2151                 'playable_in_embed': True,
2152                 'live_status': 'not_live',
2153                 'channel': 'l\'Or Vert asbl',
2154                 'channel_follower_count': int,
2155                 'uploader': 'l\'Or Vert asbl',
2156                 'uploader_url': 'https://www.youtube.com/@ElevageOrVert',
2157                 'uploader_id': '@ElevageOrVert',
2158             },
2159             'params': {
2160                 'skip_download': True,
2161             },
2162         },
2163         {
2164             # with '};' inside yt initial data (see [1])
2165             # see [2] for an example with '};' inside ytInitialPlayerResponse
2166             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2167             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2168             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2169             'info_dict': {
2170                 'id': 'CHqg6qOn4no',
2171                 'ext': 'mp4',
2172                 'title': 'Part 77   Sort a list of simple types in c#',
2173                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2174                 'upload_date': '20130831',
2175                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2176                 'like_count': int,
2177                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2178                 'live_status': 'not_live',
2179                 'categories': ['Education'],
2180                 'availability': 'public',
2181                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2182                 'tags': 'count:12',
2183                 'playable_in_embed': True,
2184                 'age_limit': 0,
2185                 'view_count': int,
2186                 'duration': 522,
2187                 'channel': 'kudvenkat',
2188                 'comment_count': int,
2189                 'channel_follower_count': int,
2190                 'chapters': list,
2191                 'uploader': 'kudvenkat',
2192                 'uploader_url': 'https://www.youtube.com/@Csharp-video-tutorialsBlogspot',
2193                 'uploader_id': '@Csharp-video-tutorialsBlogspot',
2194                 'channel_is_verified': True,
2195                 'heatmap': 'count:100',
2196             },
2197             'params': {
2198                 'skip_download': True,
2199             },
2200         },
2201         {
2202             # another example of '};' in ytInitialData
2203             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2204             'only_matching': True,
2205         },
2206         {
2207             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2208             'only_matching': True,
2209         },
2210         {
2211             # https://github.com/ytdl-org/youtube-dl/pull/28094
2212             'url': 'OtqTfy26tG0',
2213             'info_dict': {
2214                 'id': 'OtqTfy26tG0',
2215                 'ext': 'mp4',
2216                 'title': 'Burn Out',
2217                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2218                 'upload_date': '20141120',
2219                 'artist': 'The Cinematic Orchestra',
2220                 'track': 'Burn Out',
2221                 'album': 'Every Day',
2222                 'like_count': int,
2223                 'live_status': 'not_live',
2224                 'alt_title': 'Burn Out',
2225                 'duration': 614,
2226                 'age_limit': 0,
2227                 'view_count': int,
2228                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2229                 'creator': 'The Cinematic Orchestra',
2230                 'channel': 'The Cinematic Orchestra',
2231                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2232                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2233                 'availability': 'public',
2234                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2235                 'categories': ['Music'],
2236                 'playable_in_embed': True,
2237                 'channel_follower_count': int,
2238                 'uploader': 'The Cinematic Orchestra',
2239                 'comment_count': int,
2240             },
2241             'params': {
2242                 'skip_download': True,
2243             },
2244         },
2245         {
2246             # controversial video, only works with bpctr when authenticated with cookies
2247             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2248             'only_matching': True,
2249         },
2250         {
2251             # controversial video, requires bpctr/contentCheckOk
2252             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2253             'info_dict': {
2254                 'id': 'SZJvDhaSDnc',
2255                 'ext': 'mp4',
2256                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2257                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2258                 'upload_date': '20140716',
2259                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2260                 'duration': 170,
2261                 'categories': ['News & Politics'],
2262                 'view_count': int,
2263                 'channel': 'CBS Mornings',
2264                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2265                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2266                 'age_limit': 18,
2267                 'availability': 'needs_auth',
2268                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2269                 'like_count': int,
2270                 'live_status': 'not_live',
2271                 'playable_in_embed': True,
2272                 'channel_follower_count': int,
2273                 'uploader': 'CBS Mornings',
2274                 'uploader_url': 'https://www.youtube.com/@CBSMornings',
2275                 'uploader_id': '@CBSMornings',
2276                 'comment_count': int,
2277                 'channel_is_verified': True,
2278             }
2279         },
2280         {
2281             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2282             'url': 'cBvYw8_A0vQ',
2283             'info_dict': {
2284                 'id': 'cBvYw8_A0vQ',
2285                 'ext': 'mp4',
2286                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2287                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2288                 'upload_date': '20201120',
2289                 'duration': 1456,
2290                 'categories': ['Travel & Events'],
2291                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2292                 'view_count': int,
2293                 'channel': 'Walk around Japan',
2294                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2295                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2296                 'age_limit': 0,
2297                 'availability': 'public',
2298                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2299                 'live_status': 'not_live',
2300                 'playable_in_embed': True,
2301                 'channel_follower_count': int,
2302                 'uploader': 'Walk around Japan',
2303                 'uploader_url': 'https://www.youtube.com/@walkaroundjapan7124',
2304                 'uploader_id': '@walkaroundjapan7124',
2305             },
2306             'params': {
2307                 'skip_download': True,
2308             },
2309         }, {
2310             # Has multiple audio streams
2311             'url': 'WaOKSUlf4TM',
2312             'only_matching': True
2313         }, {
2314             # Requires Premium: has format 141 when requested using YTM url
2315             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2316             'only_matching': True
2317         }, {
2318             # multiple subtitles with same lang_code
2319             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2320             'only_matching': True,
2321         }, {
2322             # Force use android client fallback
2323             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2324             'info_dict': {
2325                 'id': 'YOelRv7fMxY',
2326                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2327                 'ext': '3gp',
2328                 'upload_date': '20210624',
2329                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2330                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2331                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2332                 'duration': 596,
2333                 'categories': ['Entertainment'],
2334                 'view_count': int,
2335                 'channel': 'colinfurze',
2336                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2337                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2338                 'age_limit': 0,
2339                 'availability': 'public',
2340                 'like_count': int,
2341                 'live_status': 'not_live',
2342                 'playable_in_embed': True,
2343                 'channel_follower_count': int,
2344                 'chapters': list,
2345                 'uploader': 'colinfurze',
2346                 'uploader_url': 'https://www.youtube.com/@colinfurze',
2347                 'uploader_id': '@colinfurze',
2348                 'comment_count': int,
2349                 'channel_is_verified': True,
2350                 'heatmap': 'count:100',
2351             },
2352             'params': {
2353                 'format': '17',  # 3gp format available on android
2354                 'extractor_args': {'youtube': {'player_client': ['android']}},
2355             },
2356         },
2357         {
2358             # Skip download of additional client configs (remix client config in this case)
2359             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2360             'only_matching': True,
2361             'params': {
2362                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2363             },
2364         }, {
2365             # shorts
2366             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2367             'only_matching': True,
2368         }, {
2369             'note': 'Storyboards',
2370             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2371             'info_dict': {
2372                 'id': '5KLPxDtMqe8',
2373                 'ext': 'mhtml',
2374                 'format_id': 'sb0',
2375                 'title': 'Your Brain is Plastic',
2376                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2377                 'upload_date': '20140324',
2378                 'like_count': int,
2379                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2380                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2381                 'view_count': int,
2382                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2383                 'playable_in_embed': True,
2384                 'tags': 'count:12',
2385                 'availability': 'public',
2386                 'channel': 'SciShow',
2387                 'live_status': 'not_live',
2388                 'duration': 248,
2389                 'categories': ['Education'],
2390                 'age_limit': 0,
2391                 'channel_follower_count': int,
2392                 'chapters': list,
2393                 'uploader': 'SciShow',
2394                 'uploader_url': 'https://www.youtube.com/@SciShow',
2395                 'uploader_id': '@SciShow',
2396                 'comment_count': int,
2397                 'channel_is_verified': True,
2398                 'heatmap': 'count:100',
2399             }, 'params': {'format': 'mhtml', 'skip_download': True}
2400         }, {
2401             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2402             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2403             'info_dict': {
2404                 'id': '2NUZ8W2llS4',
2405                 'ext': 'mp4',
2406                 'title': 'The NP that test your phone performance 🙂',
2407                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2408                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2409                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2410                 'duration': 21,
2411                 'view_count': int,
2412                 'age_limit': 0,
2413                 'categories': ['Gaming'],
2414                 'tags': 'count:23',
2415                 'playable_in_embed': True,
2416                 'live_status': 'not_live',
2417                 'upload_date': '20220103',
2418                 'like_count': int,
2419                 'availability': 'public',
2420                 'channel': 'Leon Nguyen',
2421                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2422                 'comment_count': int,
2423                 'channel_follower_count': int,
2424                 'uploader': 'Leon Nguyen',
2425                 'uploader_url': 'https://www.youtube.com/@LeonNguyen',
2426                 'uploader_id': '@LeonNguyen',
2427                 'heatmap': 'count:100',
2428             }
2429         }, {
2430             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2431             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2432             'info_dict': {
2433                 'id': '2NUZ8W2llS4',
2434                 'ext': 'mp4',
2435                 'title': 'The NP that test your phone performance 🙂',
2436                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2437                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2438                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2439                 'duration': 21,
2440                 'view_count': int,
2441                 'age_limit': 0,
2442                 'categories': ['Gaming'],
2443                 'tags': 'count:23',
2444                 'playable_in_embed': True,
2445                 'live_status': 'not_live',
2446                 'upload_date': '20220102',
2447                 'like_count': int,
2448                 'availability': 'public',
2449                 'channel': 'Leon Nguyen',
2450                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2451                 'comment_count': int,
2452                 'channel_follower_count': int,
2453                 'uploader': 'Leon Nguyen',
2454                 'uploader_url': 'https://www.youtube.com/@LeonNguyen',
2455                 'uploader_id': '@LeonNguyen',
2456                 'heatmap': 'count:100',
2457             },
2458             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2459         }, {
2460             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2461             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2462             'info_dict': {
2463                 'id': 'mzZzzBU6lrM',
2464                 'ext': 'mp4',
2465                 'title': 'I Met GeorgeNotFound In Real Life...',
2466                 'description': 'md5:978296ec9783a031738b684d4ebf302d',
2467                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2468                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2469                 'duration': 955,
2470                 'view_count': int,
2471                 'age_limit': 0,
2472                 'categories': ['Entertainment'],
2473                 'tags': 'count:26',
2474                 'playable_in_embed': True,
2475                 'live_status': 'not_live',
2476                 'release_timestamp': 1641172509,
2477                 'release_date': '20220103',
2478                 'upload_date': '20220103',
2479                 'like_count': int,
2480                 'availability': 'public',
2481                 'channel': 'Quackity',
2482                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2483                 'channel_follower_count': int,
2484                 'uploader': 'Quackity',
2485                 'uploader_id': '@Quackity',
2486                 'uploader_url': 'https://www.youtube.com/@Quackity',
2487                 'comment_count': int,
2488                 'channel_is_verified': True,
2489                 'heatmap': 'count:100',
2490             }
2491         },
2492         {   # continuous livestream. Microformat upload date should be preferred.
2493             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2494             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2495             'info_dict': {
2496                 'id': 'kgx4WGK0oNU',
2497                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2498                 'ext': 'mp4',
2499                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2500                 'availability': 'public',
2501                 'age_limit': 0,
2502                 'release_timestamp': 1637975704,
2503                 'upload_date': '20210619',
2504                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2505                 'live_status': 'is_live',
2506                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2507                 'channel': 'Abao in Tokyo',
2508                 'channel_follower_count': int,
2509                 'release_date': '20211127',
2510                 'tags': 'count:39',
2511                 'categories': ['People & Blogs'],
2512                 'like_count': int,
2513                 'view_count': int,
2514                 'playable_in_embed': True,
2515                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2516                 'concurrent_view_count': int,
2517                 'uploader': 'Abao in Tokyo',
2518                 'uploader_url': 'https://www.youtube.com/@abaointokyo',
2519                 'uploader_id': '@abaointokyo',
2520             },
2521             'params': {'skip_download': True}
2522         }, {
2523             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2524             'info_dict': {
2525                 'id': 'tjjjtzRLHvA',
2526                 'ext': 'mp4',
2527                 'title': 'ハッシュタグ無し };if window.ytcsi',
2528                 'upload_date': '20220323',
2529                 'like_count': int,
2530                 'availability': 'unlisted',
2531                 'channel': 'Lesmiscore',
2532                 'thumbnail': r're:^https?://.*\.jpg',
2533                 'age_limit': 0,
2534                 'categories': ['Music'],
2535                 'view_count': int,
2536                 'description': '',
2537                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2538                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2539                 'live_status': 'not_live',
2540                 'playable_in_embed': True,
2541                 'channel_follower_count': int,
2542                 'duration': 6,
2543                 'tags': [],
2544                 'uploader_id': '@lesmiscore',
2545                 'uploader': 'Lesmiscore',
2546                 'uploader_url': 'https://www.youtube.com/@lesmiscore',
2547             }
2548         }, {
2549             # Prefer primary title+description language metadata by default
2550             # Do not prefer translated description if primary is empty
2551             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2552             'info_dict': {
2553                 'id': 'el3E4MbxRqQ',
2554                 'ext': 'mp4',
2555                 'title': 'dlp test video 2 - primary sv no desc',
2556                 'description': '',
2557                 'channel': 'cole-dlp-test-acc',
2558                 'tags': [],
2559                 'view_count': int,
2560                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2561                 'like_count': int,
2562                 'playable_in_embed': True,
2563                 'availability': 'unlisted',
2564                 'thumbnail': r're:^https?://.*\.jpg',
2565                 'age_limit': 0,
2566                 'duration': 5,
2567                 'live_status': 'not_live',
2568                 'upload_date': '20220908',
2569                 'categories': ['People & Blogs'],
2570                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2571                 'uploader_url': 'https://www.youtube.com/@coletdjnz',
2572                 'uploader_id': '@coletdjnz',
2573                 'uploader': 'cole-dlp-test-acc',
2574             },
2575             'params': {'skip_download': True}
2576         }, {
2577             # Extractor argument: prefer translated title+description
2578             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2579             'info_dict': {
2580                 'id': 'gHKT4uU8Zng',
2581                 'ext': 'mp4',
2582                 'channel': 'cole-dlp-test-acc',
2583                 'tags': [],
2584                 'duration': 5,
2585                 'live_status': 'not_live',
2586                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2587                 'upload_date': '20220728',
2588                 'view_count': int,
2589                 'categories': ['People & Blogs'],
2590                 'thumbnail': r're:^https?://.*\.jpg',
2591                 'title': 'dlp test video title translated (fr)',
2592                 'availability': 'public',
2593                 'age_limit': 0,
2594                 'description': 'dlp test video description translated (fr)',
2595                 'playable_in_embed': True,
2596                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2597                 'uploader_url': 'https://www.youtube.com/@coletdjnz',
2598                 'uploader_id': '@coletdjnz',
2599                 'uploader': 'cole-dlp-test-acc',
2600             },
2601             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2602             'expected_warnings': [r'Preferring "fr" translated fields'],
2603         }, {
2604             'note': '6 channel audio',
2605             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2606             'only_matching': True,
2607         }, {
2608             'note': 'Multiple HLS formats with same itag',
2609             'url': 'https://www.youtube.com/watch?v=kX3nB4PpJko',
2610             'info_dict': {
2611                 'id': 'kX3nB4PpJko',
2612                 'ext': 'mp4',
2613                 'categories': ['Entertainment'],
2614                 'description': 'md5:e8031ff6e426cdb6a77670c9b81f6fa6',
2615                 'live_status': 'not_live',
2616                 'duration': 937,
2617                 'channel_follower_count': int,
2618                 'thumbnail': 'https://i.ytimg.com/vi_webp/kX3nB4PpJko/maxresdefault.webp',
2619                 'title': 'Last To Take Hand Off Jet, Keeps It!',
2620                 'channel': 'MrBeast',
2621                 'playable_in_embed': True,
2622                 'view_count': int,
2623                 'upload_date': '20221112',
2624                 'channel_url': 'https://www.youtube.com/channel/UCX6OQ3DkcsbYNE6H8uQQuVA',
2625                 'age_limit': 0,
2626                 'availability': 'public',
2627                 'channel_id': 'UCX6OQ3DkcsbYNE6H8uQQuVA',
2628                 'like_count': int,
2629                 'tags': [],
2630                 'uploader': 'MrBeast',
2631                 'uploader_url': 'https://www.youtube.com/@MrBeast',
2632                 'uploader_id': '@MrBeast',
2633                 'comment_count': int,
2634                 'channel_is_verified': True,
2635                 'heatmap': 'count:100',
2636             },
2637             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
2638         }, {
2639             'note': 'Audio formats with Dynamic Range Compression',
2640             'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
2641             'info_dict': {
2642                 'id': 'Tq92D6wQ1mg',
2643                 'ext': 'webm',
2644                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
2645                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2646                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2647                 'channel_follower_count': int,
2648                 'description': 'md5:17eccca93a786d51bc67646756894066',
2649                 'upload_date': '20191228',
2650                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
2651                 'playable_in_embed': True,
2652                 'like_count': int,
2653                 'categories': ['Entertainment'],
2654                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
2655                 'age_limit': 18,
2656                 'channel': 'Projekt Melody',
2657                 'view_count': int,
2658                 'availability': 'needs_auth',
2659                 'comment_count': int,
2660                 'live_status': 'not_live',
2661                 'duration': 106,
2662                 'uploader': 'Projekt Melody',
2663                 'uploader_id': '@ProjektMelody',
2664                 'uploader_url': 'https://www.youtube.com/@ProjektMelody',
2665             },
2666             'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
2667         },
2668         {
2669             'url': 'https://www.youtube.com/live/qVv6vCqciTM',
2670             'info_dict': {
2671                 'id': 'qVv6vCqciTM',
2672                 'ext': 'mp4',
2673                 'age_limit': 0,
2674                 'comment_count': int,
2675                 'chapters': 'count:13',
2676                 'upload_date': '20221223',
2677                 'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
2678                 'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
2679                 'like_count': int,
2680                 'release_date': '20221223',
2681                 'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
2682                 'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
2683                 'view_count': int,
2684                 'playable_in_embed': True,
2685                 'duration': 4438,
2686                 'availability': 'public',
2687                 'channel_follower_count': int,
2688                 'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
2689                 'categories': ['Entertainment'],
2690                 'live_status': 'was_live',
2691                 'release_timestamp': 1671793345,
2692                 'channel': 'さなちゃんねる',
2693                 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
2694                 'uploader': 'さなちゃんねる',
2695                 'uploader_url': 'https://www.youtube.com/@sana_natori',
2696                 'uploader_id': '@sana_natori',
2697                 'channel_is_verified': True,
2698                 'heatmap': 'count:100',
2699             },
2700         },
2701         {
2702             # Fallbacks when webpage and web client is unavailable
2703             'url': 'https://www.youtube.com/watch?v=wSSmNUl9Snw',
2704             'info_dict': {
2705                 'id': 'wSSmNUl9Snw',
2706                 'ext': 'mp4',
2707                 # 'categories': ['Science & Technology'],
2708                 'view_count': int,
2709                 'chapters': 'count:2',
2710                 'channel': 'Scott Manley',
2711                 'like_count': int,
2712                 'age_limit': 0,
2713                 # 'availability': 'public',
2714                 'channel_follower_count': int,
2715                 'live_status': 'not_live',
2716                 'upload_date': '20170831',
2717                 'duration': 682,
2718                 'tags': 'count:8',
2719                 'uploader_url': 'https://www.youtube.com/@scottmanley',
2720                 'description': 'md5:f4bed7b200404b72a394c2f97b782c02',
2721                 'uploader': 'Scott Manley',
2722                 'uploader_id': '@scottmanley',
2723                 'title': 'The Computer Hack That Saved Apollo 14',
2724                 'channel_id': 'UCxzC4EngIsMrPmbm6Nxvb-A',
2725                 'thumbnail': r're:^https?://.*\.webp',
2726                 'channel_url': 'https://www.youtube.com/channel/UCxzC4EngIsMrPmbm6Nxvb-A',
2727                 'playable_in_embed': True,
2728                 'comment_count': int,
2729                 'channel_is_verified': True,
2730                 'heatmap': 'count:100',
2731             },
2732             'params': {
2733                 'extractor_args': {'youtube': {'player_client': ['android'], 'player_skip': ['webpage']}},
2734             },
2735         },
2736     ]
2737
2738     _WEBPAGE_TESTS = [
2739         # YouTube <object> embed
2740         {
2741             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2742             'md5': '873c81d308b979f0e23ee7e620b312a3',
2743             'info_dict': {
2744                 'id': 'msN87y-iEx0',
2745                 'ext': 'mp4',
2746                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2747                 'upload_date': '20080526',
2748                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2749                 'age_limit': 0,
2750                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2751                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2752                 'playable_in_embed': True,
2753                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2754                 'like_count': int,
2755                 'comment_count': int,
2756                 'channel': 'Christopher Sykes',
2757                 'live_status': 'not_live',
2758                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2759                 'availability': 'public',
2760                 'duration': 195,
2761                 'view_count': int,
2762                 'categories': ['Science & Technology'],
2763                 'channel_follower_count': int,
2764                 'uploader': 'Christopher Sykes',
2765                 'uploader_url': 'https://www.youtube.com/@ChristopherSykesDocumentaries',
2766                 'uploader_id': '@ChristopherSykesDocumentaries',
2767                 'heatmap': 'count:100',
2768             },
2769             'params': {
2770                 'skip_download': True,
2771             }
2772         },
2773     ]
2774
2775     @classmethod
2776     def suitable(cls, url):
2777         from ..utils import parse_qs
2778
2779         qs = parse_qs(url)
2780         if qs.get('list', [None])[0]:
2781             return False
2782         return super().suitable(url)
2783
2784     def __init__(self, *args, **kwargs):
2785         super().__init__(*args, **kwargs)
2786         self._code_cache = {}
2787         self._player_cache = {}
2788
2789     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2790         lock = threading.Lock()
2791         start_time = time.time()
2792         formats = [f for f in formats if f.get('is_from_start')]
2793
2794         def refetch_manifest(format_id, delay):
2795             nonlocal formats, start_time, is_live
2796             if time.time() <= start_time + delay:
2797                 return
2798
2799             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2800             video_details = traverse_obj(prs, (..., 'videoDetails'), expected_type=dict)
2801             microformats = traverse_obj(
2802                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2803                 expected_type=dict)
2804             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2805             is_live = live_status == 'is_live'
2806             start_time = time.time()
2807
2808         def mpd_feed(format_id, delay):
2809             """
2810             @returns (manifest_url, manifest_stream_number, is_live) or None
2811             """
2812             for retry in self.RetryManager(fatal=False):
2813                 with lock:
2814                     refetch_manifest(format_id, delay)
2815
2816                 f = next((f for f in formats if f['format_id'] == format_id), None)
2817                 if not f:
2818                     if not is_live:
2819                         retry.error = f'{video_id}: Video is no longer live'
2820                     else:
2821                         retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
2822                     continue
2823                 return f['manifest_url'], f['manifest_stream_number'], is_live
2824             return None
2825
2826         for f in formats:
2827             f['is_live'] = is_live
2828             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2829                                     live_start_time, mpd_feed, not is_live and f.copy())
2830             if is_live:
2831                 f['fragments'] = gen
2832                 f['protocol'] = 'http_dash_segments_generator'
2833             else:
2834                 f['fragments'] = LazyList(gen({}))
2835                 del f['is_from_start']
2836
2837     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2838         FETCH_SPAN, MAX_DURATION = 5, 432000
2839
2840         mpd_url, stream_number, is_live = None, None, True
2841
2842         begin_index = 0
2843         download_start_time = ctx.get('start') or time.time()
2844
2845         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2846         if lack_early_segments:
2847             self.report_warning(bug_reports_message(
2848                 'Starting download from the last 120 hours of the live stream since '
2849                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2850             lack_early_segments = True
2851
2852         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2853         fragments, fragment_base_url = None, None
2854
2855         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2856             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2857             # Obtain from MPD's maximum seq value
2858             old_mpd_url = mpd_url
2859             last_error = ctx.pop('last_error', None)
2860             expire_fast = immediate or last_error and isinstance(last_error, HTTPError) and last_error.status == 403
2861             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2862                                                or (mpd_url, stream_number, False))
2863             if not refresh_sequence:
2864                 if expire_fast and not is_live:
2865                     return False, last_seq
2866                 elif old_mpd_url == mpd_url:
2867                     return True, last_seq
2868             if manifestless_orig_fmt:
2869                 fmt_info = manifestless_orig_fmt
2870             else:
2871                 try:
2872                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2873                         mpd_url, None, note=False, errnote=False, fatal=False)
2874                 except ExtractorError:
2875                     fmts = None
2876                 if not fmts:
2877                     no_fragment_score += 2
2878                     return False, last_seq
2879                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2880             fragments = fmt_info['fragments']
2881             fragment_base_url = fmt_info['fragment_base_url']
2882             assert fragment_base_url
2883
2884             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2885             return True, _last_seq
2886
2887         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2888         while is_live:
2889             fetch_time = time.time()
2890             if no_fragment_score > 30:
2891                 return
2892             if last_segment_url:
2893                 # Obtain from "X-Head-Seqnum" header value from each segment
2894                 try:
2895                     urlh = self._request_webpage(
2896                         last_segment_url, None, note=False, errnote=False, fatal=False)
2897                 except ExtractorError:
2898                     urlh = None
2899                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2900                 if last_seq is None:
2901                     no_fragment_score += 2
2902                     last_segment_url = None
2903                     continue
2904             else:
2905                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2906                 no_fragment_score += 2
2907                 if not should_continue:
2908                     continue
2909
2910             if known_idx > last_seq:
2911                 last_segment_url = None
2912                 continue
2913
2914             last_seq += 1
2915
2916             if begin_index < 0 and known_idx < 0:
2917                 # skip from the start when it's negative value
2918                 known_idx = last_seq + begin_index
2919             if lack_early_segments:
2920                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2921             try:
2922                 for idx in range(known_idx, last_seq):
2923                     # do not update sequence here or you'll get skipped some part of it
2924                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2925                     if not should_continue:
2926                         known_idx = idx - 1
2927                         raise ExtractorError('breaking out of outer loop')
2928                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2929                     yield {
2930                         'url': last_segment_url,
2931                         'fragment_count': last_seq,
2932                     }
2933                 if known_idx == last_seq:
2934                     no_fragment_score += 5
2935                 else:
2936                     no_fragment_score = 0
2937                 known_idx = last_seq
2938             except ExtractorError:
2939                 continue
2940
2941             if manifestless_orig_fmt:
2942                 # Stop at the first iteration if running for post-live manifestless;
2943                 # fragment count no longer increase since it starts
2944                 break
2945
2946             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2947
2948     def _extract_player_url(self, *ytcfgs, webpage=None):
2949         player_url = traverse_obj(
2950             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2951             get_all=False, expected_type=str)
2952         if not player_url:
2953             return
2954         return urljoin('https://www.youtube.com', player_url)
2955
2956     def _download_player_url(self, video_id, fatal=False):
2957         res = self._download_webpage(
2958             'https://www.youtube.com/iframe_api',
2959             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2960         if res:
2961             player_version = self._search_regex(
2962                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2963             if player_version:
2964                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2965
2966     def _signature_cache_id(self, example_sig):
2967         """ Return a string representation of a signature """
2968         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2969
2970     @classmethod
2971     def _extract_player_info(cls, player_url):
2972         for player_re in cls._PLAYER_INFO_RE:
2973             id_m = re.search(player_re, player_url)
2974             if id_m:
2975                 break
2976         else:
2977             raise ExtractorError('Cannot identify player %r' % player_url)
2978         return id_m.group('id')
2979
2980     def _load_player(self, video_id, player_url, fatal=True):
2981         player_id = self._extract_player_info(player_url)
2982         if player_id not in self._code_cache:
2983             code = self._download_webpage(
2984                 player_url, video_id, fatal=fatal,
2985                 note='Downloading player ' + player_id,
2986                 errnote='Download of %s failed' % player_url)
2987             if code:
2988                 self._code_cache[player_id] = code
2989         return self._code_cache.get(player_id)
2990
2991     def _extract_signature_function(self, video_id, player_url, example_sig):
2992         player_id = self._extract_player_info(player_url)
2993
2994         # Read from filesystem cache
2995         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2996         assert os.path.basename(func_id) == func_id
2997
2998         self.write_debug(f'Extracting signature function {func_id}')
2999         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
3000
3001         if not cache_spec:
3002             code = self._load_player(video_id, player_url)
3003         if code:
3004             res = self._parse_sig_js(code)
3005             test_string = ''.join(map(chr, range(len(example_sig))))
3006             cache_spec = [ord(c) for c in res(test_string)]
3007             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
3008
3009         return lambda s: ''.join(s[i] for i in cache_spec)
3010
3011     def _print_sig_code(self, func, example_sig):
3012         if not self.get_param('youtube_print_sig_code'):
3013             return
3014
3015         def gen_sig_code(idxs):
3016             def _genslice(start, end, step):
3017                 starts = '' if start == 0 else str(start)
3018                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
3019                 steps = '' if step == 1 else (':%d' % step)
3020                 return f's[{starts}{ends}{steps}]'
3021
3022             step = None
3023             # Quelch pyflakes warnings - start will be set when step is set
3024             start = '(Never used)'
3025             for i, prev in zip(idxs[1:], idxs[:-1]):
3026                 if step is not None:
3027                     if i - prev == step:
3028                         continue
3029                     yield _genslice(start, prev, step)
3030                     step = None
3031                     continue
3032                 if i - prev in [-1, 1]:
3033                     step = i - prev
3034                     start = prev
3035                     continue
3036                 else:
3037                     yield 's[%d]' % prev
3038             if step is None:
3039                 yield 's[%d]' % i
3040             else:
3041                 yield _genslice(start, i, step)
3042
3043         test_string = ''.join(map(chr, range(len(example_sig))))
3044         cache_res = func(test_string)
3045         cache_spec = [ord(c) for c in cache_res]
3046         expr_code = ' + '.join(gen_sig_code(cache_spec))
3047         signature_id_tuple = '(%s)' % (
3048             ', '.join(str(len(p)) for p in example_sig.split('.')))
3049         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
3050                 '    return %s\n') % (signature_id_tuple, expr_code)
3051         self.to_screen('Extracted signature function:\n' + code)
3052
3053     def _parse_sig_js(self, jscode):
3054         funcname = self._search_regex(
3055             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3056              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3057              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
3058              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
3059              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
3060              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
3061              # Obsolete patterns
3062              r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3063              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
3064              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3065              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3066              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3067              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
3068             jscode, 'Initial JS player signature function name', group='sig')
3069
3070         jsi = JSInterpreter(jscode)
3071         initial_function = jsi.extract_function(funcname)
3072         return lambda s: initial_function([s])
3073
3074     def _cached(self, func, *cache_id):
3075         def inner(*args, **kwargs):
3076             if cache_id not in self._player_cache:
3077                 try:
3078                     self._player_cache[cache_id] = func(*args, **kwargs)
3079                 except ExtractorError as e:
3080                     self._player_cache[cache_id] = e
3081                 except Exception as e:
3082                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
3083
3084             ret = self._player_cache[cache_id]
3085             if isinstance(ret, Exception):
3086                 raise ret
3087             return ret
3088         return inner
3089
3090     def _decrypt_signature(self, s, video_id, player_url):
3091         """Turn the encrypted s field into a working signature"""
3092         extract_sig = self._cached(
3093             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
3094         func = extract_sig(video_id, player_url, s)
3095         self._print_sig_code(func, s)
3096         return func(s)
3097
3098     def _decrypt_nsig(self, s, video_id, player_url):
3099         """Turn the encrypted n field into a working signature"""
3100         if player_url is None:
3101             raise ExtractorError('Cannot decrypt nsig without player_url')
3102         player_url = urljoin('https://www.youtube.com', player_url)
3103
3104         try:
3105             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
3106         except ExtractorError as e:
3107             raise ExtractorError('Unable to extract nsig function code', cause=e)
3108         if self.get_param('youtube_print_sig_code'):
3109             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
3110
3111         try:
3112             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
3113             ret = extract_nsig(jsi, func_code)(s)
3114         except JSInterpreter.Exception as e:
3115             try:
3116                 jsi = PhantomJSwrapper(self, timeout=5000)
3117             except ExtractorError:
3118                 raise e
3119             self.report_warning(
3120                 f'Native nsig extraction failed: Trying with PhantomJS\n'
3121                 f'         n = {s} ; player = {player_url}', video_id)
3122             self.write_debug(e, only_once=True)
3123
3124             args, func_body = func_code
3125             ret = jsi.execute(
3126                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
3127                 video_id=video_id, note='Executing signature code').strip()
3128
3129         self.write_debug(f'Decrypted nsig {s} => {ret}')
3130         return ret
3131
3132     def _extract_n_function_name(self, jscode):
3133         funcname, idx = self._search_regex(
3134             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
3135             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
3136         if not idx:
3137             return funcname
3138
3139         return json.loads(js_to_json(self._search_regex(
3140             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\])\s*[,;]', jscode,
3141             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
3142
3143     def _extract_n_function_code(self, video_id, player_url):
3144         player_id = self._extract_player_info(player_url)
3145         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
3146         jscode = func_code or self._load_player(video_id, player_url)
3147         jsi = JSInterpreter(jscode)
3148
3149         if func_code:
3150             return jsi, player_id, func_code
3151
3152         func_name = self._extract_n_function_name(jscode)
3153
3154         # For redundancy
3155         func_code = self._search_regex(
3156             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
3157                      # NB: The end of the regex is intentionally kept strict
3158                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
3159             jscode, 'nsig function', group=('var', 'code'), default=None)
3160         if func_code:
3161             func_code = ([func_code[0]], func_code[1])
3162         else:
3163             self.write_debug('Extracting nsig function with jsinterp')
3164             func_code = jsi.extract_function_code(func_name)
3165
3166         self.cache.store('youtube-nsig', player_id, func_code)
3167         return jsi, player_id, func_code
3168
3169     def _extract_n_function_from_code(self, jsi, func_code):
3170         func = jsi.extract_function_from_code(*func_code)
3171
3172         def extract_nsig(s):
3173             try:
3174                 ret = func([s])
3175             except JSInterpreter.Exception:
3176                 raise
3177             except Exception as e:
3178                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
3179
3180             if ret.startswith('enhanced_except_'):
3181                 raise JSInterpreter.Exception('Signature function returned an exception')
3182             return ret
3183
3184         return extract_nsig
3185
3186     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
3187         """
3188         Extract signatureTimestamp (sts)
3189         Required to tell API what sig/player version is in use.
3190         """
3191         sts = None
3192         if isinstance(ytcfg, dict):
3193             sts = int_or_none(ytcfg.get('STS'))
3194
3195         if not sts:
3196             # Attempt to extract from player
3197             if player_url is None:
3198                 error_msg = 'Cannot extract signature timestamp without player_url.'
3199                 if fatal:
3200                     raise ExtractorError(error_msg)
3201                 self.report_warning(error_msg)
3202                 return
3203             code = self._load_player(video_id, player_url, fatal=fatal)
3204             if code:
3205                 sts = int_or_none(self._search_regex(
3206                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
3207                     'JS player signature timestamp', group='sts', fatal=fatal))
3208         return sts
3209
3210     def _mark_watched(self, video_id, player_responses):
3211         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
3212             label = 'fully ' if is_full else ''
3213             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
3214                             expected_type=url_or_none)
3215             if not url:
3216                 self.report_warning(f'Unable to mark {label}watched')
3217                 return
3218             parsed_url = urllib.parse.urlparse(url)
3219             qs = urllib.parse.parse_qs(parsed_url.query)
3220
3221             # cpn generation algorithm is reverse engineered from base.js.
3222             # In fact it works even with dummy cpn.
3223             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
3224             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
3225
3226             # # more consistent results setting it to right before the end
3227             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
3228
3229             qs.update({
3230                 'ver': ['2'],
3231                 'cpn': [cpn],
3232                 'cmt': video_length,
3233                 'el': 'detailpage',  # otherwise defaults to "shorts"
3234             })
3235
3236             if is_full:
3237                 # these seem to mark watchtime "history" in the real world
3238                 # they're required, so send in a single value
3239                 qs.update({
3240                     'st': 0,
3241                     'et': video_length,
3242                 })
3243
3244             url = urllib.parse.urlunparse(
3245                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
3246
3247             self._download_webpage(
3248                 url, video_id, f'Marking {label}watched',
3249                 'Unable to mark watched', fatal=False)
3250
3251     @classmethod
3252     def _extract_from_webpage(cls, url, webpage):
3253         # Invidious Instances
3254         # https://github.com/yt-dlp/yt-dlp/issues/195
3255         # https://github.com/iv-org/invidious/pull/1730
3256         mobj = re.search(
3257             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
3258             webpage)
3259         if mobj:
3260             yield cls.url_result(mobj.group('url'), cls)
3261             raise cls.StopExtraction()
3262
3263         yield from super()._extract_from_webpage(url, webpage)
3264
3265         # lazyYT YouTube embed
3266         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
3267             yield cls.url_result(unescapeHTML(id_), cls, id_)
3268
3269         # Wordpress "YouTube Video Importer" plugin
3270         for m in re.findall(r'''(?x)<div[^>]+
3271                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
3272                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
3273             yield cls.url_result(m[-1], cls, m[-1])
3274
3275     @classmethod
3276     def extract_id(cls, url):
3277         video_id = cls.get_temp_id(url)
3278         if not video_id:
3279             raise ExtractorError(f'Invalid URL: {url}')
3280         return video_id
3281
3282     def _extract_chapters_from_json(self, data, duration):
3283         chapter_list = traverse_obj(
3284             data, (
3285                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3286                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3287             ), expected_type=list)
3288
3289         return self._extract_chapters_helper(
3290             chapter_list,
3291             start_function=lambda chapter: float_or_none(
3292                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3293             title_function=lambda chapter: traverse_obj(
3294                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3295             duration=duration)
3296
3297     def _extract_chapters_from_engagement_panel(self, data, duration):
3298         content_list = traverse_obj(
3299             data,
3300             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3301             expected_type=list)
3302         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3303         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3304
3305         return next(filter(None, (
3306             self._extract_chapters_helper(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3307                                           chapter_time, chapter_title, duration)
3308             for contents in content_list)), [])
3309
3310     def _extract_heatmap(self, data):
3311         return traverse_obj(data, (
3312             'frameworkUpdates', 'entityBatchUpdate', 'mutations',
3313             lambda _, v: v['payload']['macroMarkersListEntity']['markersList']['markerType'] == 'MARKER_TYPE_HEATMAP',
3314             'payload', 'macroMarkersListEntity', 'markersList', 'markers', ..., {
3315                 'start_time': ('startMillis', {functools.partial(float_or_none, scale=1000)}),
3316                 'end_time': {lambda x: (int(x['startMillis']) + int(x['durationMillis'])) / 1000},
3317                 'value': ('intensityScoreNormalized', {float_or_none}),
3318             })) or None
3319
3320     def _extract_comment(self, comment_renderer, parent=None):
3321         comment_id = comment_renderer.get('commentId')
3322         if not comment_id:
3323             return
3324
3325         info = {
3326             'id': comment_id,
3327             'text': self._get_text(comment_renderer, 'contentText'),
3328             'like_count': self._get_count(comment_renderer, 'voteCount'),
3329             'author_id': traverse_obj(comment_renderer, ('authorEndpoint', 'browseEndpoint', 'browseId', {self.ucid_or_none})),
3330             'author': self._get_text(comment_renderer, 'authorText'),
3331             'author_thumbnail': traverse_obj(comment_renderer, ('authorThumbnail', 'thumbnails', -1, 'url', {url_or_none})),
3332             'parent': parent or 'root',
3333         }
3334
3335         # Timestamp is an estimate calculated from the current time and time_text
3336         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3337         timestamp = self._parse_time_text(time_text)
3338
3339         info.update({
3340             # FIXME: non-standard, but we need a way of showing that it is an estimate.
3341             '_time_text': time_text,
3342             'timestamp': timestamp,
3343         })
3344
3345         info['author_url'] = urljoin(
3346             'https://www.youtube.com', traverse_obj(comment_renderer, ('authorEndpoint', (
3347                 ('browseEndpoint', 'canonicalBaseUrl'), ('commandMetadata', 'webCommandMetadata', 'url'))),
3348                 expected_type=str, get_all=False))
3349
3350         author_is_uploader = traverse_obj(comment_renderer, 'authorIsChannelOwner')
3351         if author_is_uploader is not None:
3352             info['author_is_uploader'] = author_is_uploader
3353
3354         comment_abr = traverse_obj(
3355             comment_renderer, ('actionButtons', 'commentActionButtonsRenderer'), expected_type=dict)
3356         if comment_abr is not None:
3357             info['is_favorited'] = 'creatorHeart' in comment_abr
3358
3359         badges = self._extract_badges([traverse_obj(comment_renderer, 'authorCommentBadge')])
3360         if self._has_badge(badges, BadgeType.VERIFIED):
3361             info['author_is_verified'] = True
3362
3363         is_pinned = traverse_obj(comment_renderer, 'pinnedCommentBadge')
3364         if is_pinned:
3365             info['is_pinned'] = True
3366
3367         return info
3368
3369     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3370
3371         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3372
3373         def extract_header(contents):
3374             _continuation = None
3375             for content in contents:
3376                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3377                 expected_comment_count = self._get_count(
3378                     comments_header_renderer, 'countText', 'commentsCount')
3379
3380                 if expected_comment_count is not None:
3381                     tracker['est_total'] = expected_comment_count
3382                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3383                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3384
3385                 sort_menu_item = try_get(
3386                     comments_header_renderer,
3387                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3388                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3389
3390                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3391                 if not _continuation:
3392                     continue
3393
3394                 sort_text = str_or_none(sort_menu_item.get('title'))
3395                 if not sort_text:
3396                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3397                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3398                 break
3399             return _continuation
3400
3401         def extract_thread(contents):
3402             if not parent:
3403                 tracker['current_page_thread'] = 0
3404             for content in contents:
3405                 if not parent and tracker['total_parent_comments'] >= max_parents:
3406                     yield
3407                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3408                 comment_renderer = get_first(
3409                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3410                     expected_type=dict, default={})
3411
3412                 comment = self._extract_comment(comment_renderer, parent)
3413                 if not comment:
3414                     continue
3415                 comment_id = comment['id']
3416                 if comment.get('is_pinned'):
3417                     tracker['pinned_comment_ids'].add(comment_id)
3418                 # Sometimes YouTube may break and give us infinite looping comments.
3419                 # See: https://github.com/yt-dlp/yt-dlp/issues/6290
3420                 if comment_id in tracker['seen_comment_ids']:
3421                     if comment_id in tracker['pinned_comment_ids'] and not comment.get('is_pinned'):
3422                         # Pinned comments may appear a second time in newest first sort
3423                         # See: https://github.com/yt-dlp/yt-dlp/issues/6712
3424                         continue
3425                     self.report_warning(
3426                         'Detected YouTube comments looping. Stopping comment extraction '
3427                         f'{"for this thread" if parent else ""} as we probably cannot get any more.')
3428                     yield
3429                 else:
3430                     tracker['seen_comment_ids'].add(comment['id'])
3431
3432                 tracker['running_total'] += 1
3433                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3434                 yield comment
3435
3436                 # Attempt to get the replies
3437                 comment_replies_renderer = try_get(
3438                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3439
3440                 if comment_replies_renderer:
3441                     tracker['current_page_thread'] += 1
3442                     comment_entries_iter = self._comment_entries(
3443                         comment_replies_renderer, ytcfg, video_id,
3444                         parent=comment.get('id'), tracker=tracker)
3445                     yield from itertools.islice(comment_entries_iter, min(
3446                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3447
3448         # Keeps track of counts across recursive calls
3449         if not tracker:
3450             tracker = dict(
3451                 running_total=0,
3452                 est_total=None,
3453                 current_page_thread=0,
3454                 total_parent_comments=0,
3455                 total_reply_comments=0,
3456                 seen_comment_ids=set(),
3457                 pinned_comment_ids=set()
3458             )
3459
3460         # TODO: Deprecated
3461         # YouTube comments have a max depth of 2
3462         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3463         if max_depth:
3464             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3465                                                 'Set max replies in the max-comments extractor argument instead')
3466         if max_depth == 1 and parent:
3467             return
3468
3469         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3470             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3471
3472         continuation = self._extract_continuation(root_continuation_data)
3473
3474         response = None
3475         is_forced_continuation = False
3476         is_first_continuation = parent is None
3477         if is_first_continuation and not continuation:
3478             # Sometimes you can get comments by generating the continuation yourself,
3479             # even if YouTube initially reports them being disabled - e.g. stories comments.
3480             # Note: if the comment section is actually disabled, YouTube may return a response with
3481             # required check_get_keys missing. So we will disable that check initially in this case.
3482             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3483             is_forced_continuation = True
3484
3485         continuation_items_path = (
3486             'onResponseReceivedEndpoints', ..., ('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems')
3487         for page_num in itertools.count(0):
3488             if not continuation:
3489                 break
3490             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3491             comment_prog_str = f"({tracker['running_total']}/~{tracker['est_total']})"
3492             if page_num == 0:
3493                 if is_first_continuation:
3494                     note_prefix = 'Downloading comment section API JSON'
3495                 else:
3496                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3497                         tracker['current_page_thread'], comment_prog_str)
3498             else:
3499                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3500                     '       ' if parent else '', ' replies' if parent else '',
3501                     page_num, comment_prog_str)
3502
3503             # Do a deep check for incomplete data as sometimes YouTube may return no comments for a continuation
3504             # Ignore check if YouTube says the comment count is 0.
3505             check_get_keys = None
3506             if not is_forced_continuation and not (tracker['est_total'] == 0 and tracker['running_total'] == 0):
3507                 check_get_keys = [[*continuation_items_path, ..., (
3508                     'commentsHeaderRenderer' if is_first_continuation else ('commentThreadRenderer', 'commentRenderer'))]]
3509             try:
3510                 response = self._extract_response(
3511                     item_id=None, query=continuation,
3512                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3513                     check_get_keys=check_get_keys)
3514             except ExtractorError as e:
3515                 # Ignore incomplete data error for replies if retries didn't work.
3516                 # This is to allow any other parent comments and comment threads to be downloaded.
3517                 # See: https://github.com/yt-dlp/yt-dlp/issues/4669
3518                 if 'incomplete data' in str(e).lower() and parent:
3519                     if self.get_param('ignoreerrors') in (True, 'only_download'):
3520                         self.report_warning(
3521                             'Received incomplete data for a comment reply thread and retrying did not help. '
3522                             'Ignoring to let other comments be downloaded. Pass --no-ignore-errors to not ignore.')
3523                         return
3524                     else:
3525                         raise ExtractorError(
3526                             'Incomplete data received for comment reply thread. '
3527                             'Pass --ignore-errors to ignore and allow rest of comments to download.',
3528                             expected=True)
3529                 raise
3530             is_forced_continuation = False
3531             continuation = None
3532             for continuation_items in traverse_obj(response, continuation_items_path, expected_type=list, default=[]):
3533                 if is_first_continuation:
3534                     continuation = extract_header(continuation_items)
3535                     is_first_continuation = False
3536                     if continuation:
3537                         break
3538                     continue
3539
3540                 for entry in extract_thread(continuation_items):
3541                     if not entry:
3542                         return
3543                     yield entry
3544                 continuation = self._extract_continuation({'contents': continuation_items})
3545                 if continuation:
3546                     break
3547
3548         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3549         if message and not parent and tracker['running_total'] == 0:
3550             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3551             raise self.CommentsDisabled
3552
3553     @staticmethod
3554     def _generate_comment_continuation(video_id):
3555         """
3556         Generates initial comment section continuation token from given video id
3557         """
3558         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3559         return base64.b64encode(token.encode()).decode()
3560
3561     def _get_comments(self, ytcfg, video_id, contents, webpage):
3562         """Entry for comment extraction"""
3563         def _real_comment_extract(contents):
3564             renderer = next((
3565                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3566                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3567             yield from self._comment_entries(renderer, ytcfg, video_id)
3568
3569         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3570         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3571
3572     @staticmethod
3573     def _get_checkok_params():
3574         return {'contentCheckOk': True, 'racyCheckOk': True}
3575
3576     @classmethod
3577     def _generate_player_context(cls, sts=None):
3578         context = {
3579             'html5Preference': 'HTML5_PREF_WANTS',
3580         }
3581         if sts is not None:
3582             context['signatureTimestamp'] = sts
3583         return {
3584             'playbackContext': {
3585                 'contentPlaybackContext': context
3586             },
3587             **cls._get_checkok_params()
3588         }
3589
3590     @staticmethod
3591     def _is_agegated(player_response):
3592         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3593             return True
3594
3595         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')))
3596         AGE_GATE_REASONS = (
3597             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3598             'age_verification_required', 'age_check_required',  # status
3599         )
3600         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3601
3602     @staticmethod
3603     def _is_unplayable(player_response):
3604         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3605
3606     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3607
3608         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3609         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3610         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3611         headers = self.generate_api_headers(
3612             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3613
3614         yt_query = {
3615             'videoId': video_id,
3616         }
3617         if _split_innertube_client(client)[0] in ('android', 'android_embedscreen'):
3618             yt_query['params'] = 'CgIIAQ=='
3619
3620         pp_arg = self._configuration_arg('player_params', [None], casesense=True)[0]
3621         if pp_arg:
3622             yt_query['params'] = pp_arg
3623
3624         yt_query.update(self._generate_player_context(sts))
3625         return self._extract_response(
3626             item_id=video_id, ep='player', query=yt_query,
3627             ytcfg=player_ytcfg, headers=headers, fatal=True,
3628             default_client=client,
3629             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3630         ) or None
3631
3632     def _get_requested_clients(self, url, smuggled_data):
3633         requested_clients = []
3634         default = ['ios', 'android', 'web']
3635         allowed_clients = sorted(
3636             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3637             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3638         for client in self._configuration_arg('player_client'):
3639             if client in allowed_clients:
3640                 requested_clients.append(client)
3641             elif client == 'default':
3642                 requested_clients.extend(default)
3643             elif client == 'all':
3644                 requested_clients.extend(allowed_clients)
3645             else:
3646                 self.report_warning(f'Skipping unsupported client {client}')
3647         if not requested_clients:
3648             requested_clients = default
3649
3650         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3651             requested_clients.extend(
3652                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3653
3654         return orderedSet(requested_clients)
3655
3656     def _invalid_player_response(self, pr, video_id):
3657         # YouTube may return a different video player response than expected.
3658         # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3659         if (pr_id := traverse_obj(pr, ('videoDetails', 'videoId'))) != video_id:
3660             return pr_id
3661
3662     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3663         initial_pr = None
3664         if webpage:
3665             initial_pr = self._search_json(
3666                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3667
3668         prs = []
3669         if initial_pr and not self._invalid_player_response(initial_pr, video_id):
3670             # Android player_response does not have microFormats which are needed for
3671             # extraction of some data. So we return the initial_pr with formats
3672             # stripped out even if not requested by the user
3673             # See: https://github.com/yt-dlp/yt-dlp/issues/501
3674             prs.append({**initial_pr, 'streamingData': None})
3675
3676         all_clients = set(clients)
3677         clients = clients[::-1]
3678
3679         def append_client(*client_names):
3680             """ Append the first client name that exists but not already used """
3681             for client_name in client_names:
3682                 actual_client = _split_innertube_client(client_name)[0]
3683                 if actual_client in INNERTUBE_CLIENTS:
3684                     if actual_client not in all_clients:
3685                         clients.append(client_name)
3686                         all_clients.add(actual_client)
3687                         return
3688
3689         tried_iframe_fallback = False
3690         player_url = None
3691         skipped_clients = {}
3692         while clients:
3693             client, base_client, variant = _split_innertube_client(clients.pop())
3694             player_ytcfg = master_ytcfg if client == 'web' else {}
3695             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3696                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3697
3698             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3699             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3700             if 'js' in self._configuration_arg('player_skip'):
3701                 require_js_player = False
3702                 player_url = None
3703
3704             if not player_url and not tried_iframe_fallback and require_js_player:
3705                 player_url = self._download_player_url(video_id)
3706                 tried_iframe_fallback = True
3707
3708             try:
3709                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3710                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3711             except ExtractorError as e:
3712                 self.report_warning(e)
3713                 continue
3714
3715             if pr_id := self._invalid_player_response(pr, video_id):
3716                 skipped_clients[client] = pr_id
3717             elif pr:
3718                 # Save client name for introspection later
3719                 name = short_client_name(client)
3720                 sd = traverse_obj(pr, ('streamingData', {dict})) or {}
3721                 sd[STREAMING_DATA_CLIENT_NAME] = name
3722                 for f in traverse_obj(sd, (('formats', 'adaptiveFormats'), ..., {dict})):
3723                     f[STREAMING_DATA_CLIENT_NAME] = name
3724                 prs.append(pr)
3725
3726             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3727             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3728                 append_client(f'{base_client}_creator')
3729             elif self._is_agegated(pr):
3730                 if variant == 'tv_embedded':
3731                     append_client(f'{base_client}_embedded')
3732                 elif not variant:
3733                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3734
3735         if skipped_clients:
3736             self.report_warning(
3737                 f'Skipping player responses from {"/".join(skipped_clients)} clients '
3738                 f'(got player responses for video "{"/".join(set(skipped_clients.values()))}" instead of "{video_id}")')
3739             if not prs:
3740                 raise ExtractorError(
3741                     'All player responses are invalid. Your IP is likely being blocked by Youtube', expected=True)
3742         elif not prs:
3743             raise ExtractorError('Failed to extract any player response')
3744         return prs, player_url
3745
3746     def _needs_live_processing(self, live_status, duration):
3747         if (live_status == 'is_live' and self.get_param('live_from_start')
3748                 or live_status == 'post_live' and (duration or 0) > 2 * 3600):
3749             return live_status
3750
3751     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3752         CHUNK_SIZE = 10 << 20
3753         itags, stream_ids = collections.defaultdict(set), []
3754         itag_qualities, res_qualities = {}, {0: None}
3755         q = qualities([
3756             # Normally tiny is the smallest video-only formats. But
3757             # audio-only formats with unknown quality may get tagged as tiny
3758             'tiny',
3759             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3760             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3761         ])
3762         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
3763         format_types = self._configuration_arg('formats')
3764         all_formats = 'duplicate' in format_types
3765         if self._configuration_arg('include_duplicate_formats'):
3766             all_formats = True
3767             self._downloader.deprecated_feature('[youtube] include_duplicate_formats extractor argument is deprecated. '
3768                                                 'Use formats=duplicate extractor argument instead')
3769
3770         def build_fragments(f):
3771             return LazyList({
3772                 'url': update_url_query(f['url'], {
3773                     'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, f["filesize"])}'
3774                 })
3775             } for range_start in range(0, f['filesize'], CHUNK_SIZE))
3776
3777         for fmt in streaming_formats:
3778             if fmt.get('targetDurationSec'):
3779                 continue
3780
3781             itag = str_or_none(fmt.get('itag'))
3782             audio_track = fmt.get('audioTrack') or {}
3783             stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
3784             if not all_formats:
3785                 if stream_id in stream_ids:
3786                     continue
3787
3788             quality = fmt.get('quality')
3789             height = int_or_none(fmt.get('height'))
3790             if quality == 'tiny' or not quality:
3791                 quality = fmt.get('audioQuality', '').lower() or quality
3792             # The 3gp format (17) in android client has a quality of "small",
3793             # but is actually worse than other formats
3794             if itag == '17':
3795                 quality = 'tiny'
3796             if quality:
3797                 if itag:
3798                     itag_qualities[itag] = quality
3799                 if height:
3800                     res_qualities[height] = quality
3801             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3802             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3803             # number of fragment that would subsequently requested with (`&sq=N`)
3804             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3805                 continue
3806
3807             fmt_url = fmt.get('url')
3808             if not fmt_url:
3809                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3810                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3811                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3812                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3813                     continue
3814                 try:
3815                     fmt_url += '&%s=%s' % (
3816                         traverse_obj(sc, ('sp', -1)) or 'signature',
3817                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3818                     )
3819                 except ExtractorError as e:
3820                     self.report_warning('Signature extraction failed: Some formats may be missing',
3821                                         video_id=video_id, only_once=True)
3822                     self.write_debug(e, only_once=True)
3823                     continue
3824
3825             query = parse_qs(fmt_url)
3826             throttled = False
3827             if query.get('n'):
3828                 try:
3829                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3830                     fmt_url = update_url_query(fmt_url, {
3831                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3832                     })
3833                 except ExtractorError as e:
3834                     phantomjs_hint = ''
3835                     if isinstance(e, JSInterpreter.Exception):
3836                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3837                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3838                     if player_url:
3839                         self.report_warning(
3840                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3841                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3842                         self.write_debug(e, only_once=True)
3843                     else:
3844                         self.report_warning(
3845                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3846                             video_id=video_id, only_once=True)
3847                     throttled = True
3848
3849             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3850             language_preference = (
3851                 10 if audio_track.get('audioIsDefault') and 10
3852                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3853                 else -1)
3854             format_duration = traverse_obj(fmt, ('approxDurationMs', {lambda x: float_or_none(x, 1000)}))
3855             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3856             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3857             # Make sure to avoid false positives with small duration differences.
3858             # E.g. __2ABJjxzNo, ySuUZEjARPY
3859             is_damaged = try_call(lambda: format_duration < duration // 2)
3860             if is_damaged:
3861                 self.report_warning(
3862                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3863
3864             client_name = fmt.get(STREAMING_DATA_CLIENT_NAME)
3865             name = fmt.get('qualityLabel') or quality.replace('audio_quality_', '') or ''
3866             fps = int_or_none(fmt.get('fps')) or 0
3867             dct = {
3868                 'asr': int_or_none(fmt.get('audioSampleRate')),
3869                 'filesize': int_or_none(fmt.get('contentLength')),
3870                 'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
3871                 'format_note': join_nonempty(
3872                     join_nonempty(audio_track.get('displayName'),
3873                                   language_preference > 0 and ' (default)', delim=''),
3874                     name, fmt.get('isDrc') and 'DRC',
3875                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3876                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3877                     throttled and 'THROTTLED', is_damaged and 'DAMAGED',
3878                     (self.get_param('verbose') or all_formats) and client_name,
3879                     delim=', '),
3880                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3881                 'source_preference': ((-10 if throttled else -5 if itag == '22' else -1)
3882                                       + (100 if 'Premium' in name else 0)),
3883                 'fps': fps if fps > 1 else None,  # For some formats, fps is wrongly returned as 1
3884                 'audio_channels': fmt.get('audioChannels'),
3885                 'height': height,
3886                 'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
3887                 'has_drm': bool(fmt.get('drmFamilies')),
3888                 'tbr': tbr,
3889                 'filesize_approx': filesize_from_tbr(tbr, format_duration),
3890                 'url': fmt_url,
3891                 'width': int_or_none(fmt.get('width')),
3892                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3893                                           'desc' if language_preference < -1 else '') or None,
3894                 'language_preference': language_preference,
3895                 # Strictly de-prioritize damaged and 3gp formats
3896                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3897             }
3898             mime_mobj = re.match(
3899                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3900             if mime_mobj:
3901                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3902                 dct.update(parse_codecs(mime_mobj.group(2)))
3903             if itag:
3904                 itags[itag].add(('https', dct.get('language')))
3905                 stream_ids.append(stream_id)
3906             single_stream = 'none' in (dct.get('acodec'), dct.get('vcodec'))
3907             if single_stream and dct.get('ext'):
3908                 dct['container'] = dct['ext'] + '_dash'
3909
3910             if (all_formats or 'dashy' in format_types) and dct['filesize']:
3911                 yield {
3912                     **dct,
3913                     'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
3914                     'protocol': 'http_dash_segments',
3915                     'fragments': build_fragments(dct),
3916                 }
3917             if all_formats or 'dashy' not in format_types:
3918                 dct['downloader_options'] = {'http_chunk_size': CHUNK_SIZE}
3919                 yield dct
3920
3921         needs_live_processing = self._needs_live_processing(live_status, duration)
3922         skip_bad_formats = 'incomplete' not in format_types
3923         if self._configuration_arg('include_incomplete_formats'):
3924             skip_bad_formats = False
3925             self._downloader.deprecated_feature('[youtube] include_incomplete_formats extractor argument is deprecated. '
3926                                                 'Use formats=incomplete extractor argument instead')
3927
3928         skip_manifests = set(self._configuration_arg('skip'))
3929         if (not self.get_param('youtube_include_hls_manifest', True)
3930                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3931                 or needs_live_processing and skip_bad_formats):
3932             skip_manifests.add('hls')
3933
3934         if not self.get_param('youtube_include_dash_manifest', True):
3935             skip_manifests.add('dash')
3936         if self._configuration_arg('include_live_dash'):
3937             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3938                                                 'Use formats=incomplete extractor argument instead')
3939         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3940             skip_manifests.add('dash')
3941
3942         def process_manifest_format(f, proto, client_name, itag):
3943             key = (proto, f.get('language'))
3944             if not all_formats and key in itags[itag]:
3945                 return False
3946             itags[itag].add(key)
3947
3948             if itag and all_formats:
3949                 f['format_id'] = f'{itag}-{proto}'
3950             elif any(p != proto for p, _ in itags[itag]):
3951                 f['format_id'] = f'{itag}-{proto}'
3952             elif itag:
3953                 f['format_id'] = itag
3954
3955             if f.get('source_preference') is None:
3956                 f['source_preference'] = -1
3957
3958             if itag in ('616', '235'):
3959                 f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ')
3960                 f['source_preference'] += 100
3961
3962             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3963             if f['quality'] == -1 and f.get('height'):
3964                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3965             if self.get_param('verbose') or all_formats:
3966                 f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
3967             if f.get('fps') and f['fps'] <= 1:
3968                 del f['fps']
3969
3970             if proto == 'hls' and f.get('has_drm'):
3971                 f['has_drm'] = 'maybe'
3972                 f['source_preference'] -= 5
3973             return True
3974
3975         subtitles = {}
3976         for sd in streaming_data:
3977             client_name = sd.get(STREAMING_DATA_CLIENT_NAME)
3978
3979             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3980             if hls_manifest_url:
3981                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3982                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3983                 subtitles = self._merge_subtitles(subs, subtitles)
3984                 for f in fmts:
3985                     if process_manifest_format(f, 'hls', client_name, self._search_regex(
3986                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3987                         yield f
3988
3989             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3990             if dash_manifest_url:
3991                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3992                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3993                 for f in formats:
3994                     if process_manifest_format(f, 'dash', client_name, f['format_id']):
3995                         f['filesize'] = int_or_none(self._search_regex(
3996                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3997                         if needs_live_processing:
3998                             f['is_from_start'] = True
3999
4000                         yield f
4001         yield subtitles
4002
4003     def _extract_storyboard(self, player_responses, duration):
4004         spec = get_first(
4005             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
4006         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
4007         if not base_url:
4008             return
4009         L = len(spec) - 1
4010         for i, args in enumerate(spec):
4011             args = args.split('#')
4012             counts = list(map(int_or_none, args[:5]))
4013             if len(args) != 8 or not all(counts):
4014                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
4015                 continue
4016             width, height, frame_count, cols, rows = counts
4017             N, sigh = args[6:]
4018
4019             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
4020             fragment_count = frame_count / (cols * rows)
4021             fragment_duration = duration / fragment_count
4022             yield {
4023                 'format_id': f'sb{i}',
4024                 'format_note': 'storyboard',
4025                 'ext': 'mhtml',
4026                 'protocol': 'mhtml',
4027                 'acodec': 'none',
4028                 'vcodec': 'none',
4029                 'url': url,
4030                 'width': width,
4031                 'height': height,
4032                 'fps': frame_count / duration,
4033                 'rows': rows,
4034                 'columns': cols,
4035                 'fragments': [{
4036                     'url': url.replace('$M', str(j)),
4037                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
4038                 } for j in range(math.ceil(fragment_count))],
4039             }
4040
4041     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
4042         webpage = None
4043         if 'webpage' not in self._configuration_arg('player_skip'):
4044             query = {'bpctr': '9999999999', 'has_verified': '1'}
4045             pp = self._configuration_arg('player_params', [None], casesense=True)[0]
4046             if pp:
4047                 query['pp'] = pp
4048             webpage = self._download_webpage(
4049                 webpage_url, video_id, fatal=False, query=query)
4050
4051         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
4052
4053         player_responses, player_url = self._extract_player_responses(
4054             self._get_requested_clients(url, smuggled_data),
4055             video_id, webpage, master_ytcfg, smuggled_data)
4056
4057         return webpage, master_ytcfg, player_responses, player_url
4058
4059     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
4060         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
4061         is_live = get_first(video_details, 'isLive')
4062         if is_live is None:
4063             is_live = get_first(live_broadcast_details, 'isLiveNow')
4064         live_content = get_first(video_details, 'isLiveContent')
4065         is_upcoming = get_first(video_details, 'isUpcoming')
4066         post_live = get_first(video_details, 'isPostLiveDvr')
4067         live_status = ('post_live' if post_live
4068                        else 'is_live' if is_live
4069                        else 'is_upcoming' if is_upcoming
4070                        else 'was_live' if live_content
4071                        else 'not_live' if False in (is_live, live_content)
4072                        else None)
4073         streaming_data = traverse_obj(player_responses, (..., 'streamingData'))
4074         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
4075         if all(f.get('has_drm') for f in formats):
4076             # If there are no formats that definitely don't have DRM, all have DRM
4077             for f in formats:
4078                 f['has_drm'] = True
4079
4080         return live_broadcast_details, live_status, streaming_data, formats, subtitles
4081
4082     def _real_extract(self, url):
4083         url, smuggled_data = unsmuggle_url(url, {})
4084         video_id = self._match_id(url)
4085
4086         base_url = self.http_scheme() + '//www.youtube.com/'
4087         webpage_url = base_url + 'watch?v=' + video_id
4088
4089         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
4090
4091         playability_statuses = traverse_obj(
4092             player_responses, (..., 'playabilityStatus'), expected_type=dict)
4093
4094         trailer_video_id = get_first(
4095             playability_statuses,
4096             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
4097             expected_type=str)
4098         if trailer_video_id:
4099             return self.url_result(
4100                 trailer_video_id, self.ie_key(), trailer_video_id)
4101
4102         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
4103                        if webpage else (lambda x: None))
4104
4105         video_details = traverse_obj(player_responses, (..., 'videoDetails'), expected_type=dict)
4106         microformats = traverse_obj(
4107             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
4108             expected_type=dict)
4109
4110         translated_title = self._get_text(microformats, (..., 'title'))
4111         video_title = (self._preferred_lang and translated_title
4112                        or get_first(video_details, 'title')  # primary
4113                        or translated_title
4114                        or search_meta(['og:title', 'twitter:title', 'title']))
4115         translated_description = self._get_text(microformats, (..., 'description'))
4116         original_description = get_first(video_details, 'shortDescription')
4117         video_description = (
4118             self._preferred_lang and translated_description
4119             # If original description is blank, it will be an empty string.
4120             # Do not prefer translated description in this case.
4121             or original_description if original_description is not None else translated_description)
4122
4123         multifeed_metadata_list = get_first(
4124             player_responses,
4125             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
4126             expected_type=str)
4127         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
4128             if self.get_param('noplaylist'):
4129                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
4130             else:
4131                 entries = []
4132                 feed_ids = []
4133                 for feed in multifeed_metadata_list.split(','):
4134                     # Unquote should take place before split on comma (,) since textual
4135                     # fields may contain comma as well (see
4136                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
4137                     feed_data = urllib.parse.parse_qs(
4138                         urllib.parse.unquote_plus(feed))
4139
4140                     def feed_entry(name):
4141                         return try_get(
4142                             feed_data, lambda x: x[name][0], str)
4143
4144                     feed_id = feed_entry('id')
4145                     if not feed_id:
4146                         continue
4147                     feed_title = feed_entry('title')
4148                     title = video_title
4149                     if feed_title:
4150                         title += ' (%s)' % feed_title
4151                     entries.append({
4152                         '_type': 'url_transparent',
4153                         'ie_key': 'Youtube',
4154                         'url': smuggle_url(
4155                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
4156                             {'force_singlefeed': True}),
4157                         'title': title,
4158                     })
4159                     feed_ids.append(feed_id)
4160                 self.to_screen(
4161                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
4162                     % (', '.join(feed_ids), video_id))
4163                 return self.playlist_result(
4164                     entries, video_id, video_title, video_description)
4165
4166         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
4167                     or int_or_none(get_first(microformats, 'lengthSeconds'))
4168                     or parse_duration(search_meta('duration')) or None)
4169
4170         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
4171             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
4172         if live_status == 'post_live':
4173             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
4174
4175         if not formats:
4176             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
4177                 self.report_drm(video_id)
4178             pemr = get_first(
4179                 playability_statuses,
4180                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
4181             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
4182             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
4183             if subreason:
4184                 if subreason == 'The uploader has not made this video available in your country.':
4185                     countries = get_first(microformats, 'availableCountries')
4186                     if not countries:
4187                         regions_allowed = search_meta('regionsAllowed')
4188                         countries = regions_allowed.split(',') if regions_allowed else None
4189                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
4190                 reason += f'. {subreason}'
4191             if reason:
4192                 self.raise_no_formats(reason, expected=True)
4193
4194         keywords = get_first(video_details, 'keywords', expected_type=list) or []
4195         if not keywords and webpage:
4196             keywords = [
4197                 unescapeHTML(m.group('content'))
4198                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
4199         for keyword in keywords:
4200             if keyword.startswith('yt:stretch='):
4201                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
4202                 if mobj:
4203                     # NB: float is intentional for forcing float division
4204                     w, h = (float(v) for v in mobj.groups())
4205                     if w > 0 and h > 0:
4206                         ratio = w / h
4207                         for f in formats:
4208                             if f.get('vcodec') != 'none':
4209                                 f['stretched_ratio'] = ratio
4210                         break
4211         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
4212         thumbnail_url = search_meta(['og:image', 'twitter:image'])
4213         if thumbnail_url:
4214             thumbnails.append({
4215                 'url': thumbnail_url,
4216             })
4217         original_thumbnails = thumbnails.copy()
4218
4219         # The best resolution thumbnails sometimes does not appear in the webpage
4220         # See: https://github.com/yt-dlp/yt-dlp/issues/340
4221         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
4222         thumbnail_names = [
4223             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
4224             # in resolution, these are not the custom thumbnail. So de-prioritize them
4225             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
4226             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
4227         ]
4228         n_thumbnail_names = len(thumbnail_names)
4229         thumbnails.extend({
4230             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
4231                 video_id=video_id, name=name, ext=ext,
4232                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
4233         } for name in thumbnail_names for ext in ('webp', 'jpg'))
4234         for thumb in thumbnails:
4235             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
4236             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
4237         self._remove_duplicate_formats(thumbnails)
4238         self._downloader._sort_thumbnails(original_thumbnails)
4239
4240         category = get_first(microformats, 'category') or search_meta('genre')
4241         channel_id = self.ucid_or_none(str_or_none(
4242             get_first(video_details, 'channelId')
4243             or get_first(microformats, 'externalChannelId')
4244             or search_meta('channelId')))
4245         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
4246
4247         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
4248         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
4249         if not duration and live_end_time and live_start_time:
4250             duration = live_end_time - live_start_time
4251
4252         needs_live_processing = self._needs_live_processing(live_status, duration)
4253
4254         def is_bad_format(fmt):
4255             if needs_live_processing and not fmt.get('is_from_start'):
4256                 return True
4257             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
4258                     and fmt.get('protocol') == 'http_dash_segments'):
4259                 return True
4260
4261         for fmt in filter(is_bad_format, formats):
4262             fmt['preference'] = (fmt.get('preference') or -1) - 10
4263             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 2 hours)', delim=' ')
4264
4265         if needs_live_processing:
4266             self._prepare_live_from_start_formats(
4267                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
4268
4269         formats.extend(self._extract_storyboard(player_responses, duration))
4270
4271         channel_handle = self.handle_from_url(owner_profile_url)
4272
4273         info = {
4274             'id': video_id,
4275             'title': video_title,
4276             'formats': formats,
4277             'thumbnails': thumbnails,
4278             # The best thumbnail that we are sure exists. Prevents unnecessary
4279             # URL checking if user don't care about getting the best possible thumbnail
4280             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
4281             'description': video_description,
4282             'channel_id': channel_id,
4283             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None),
4284             'duration': duration,
4285             'view_count': int_or_none(
4286                 get_first((video_details, microformats), (..., 'viewCount'))
4287                 or search_meta('interactionCount')),
4288             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
4289             'age_limit': 18 if (
4290                 get_first(microformats, 'isFamilySafe') is False
4291                 or search_meta('isFamilyFriendly') == 'false'
4292                 or search_meta('og:restrictions:age') == '18+') else 0,
4293             'webpage_url': webpage_url,
4294             'categories': [category] if category else None,
4295             'tags': keywords,
4296             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
4297             'live_status': live_status,
4298             'release_timestamp': live_start_time,
4299             '_format_sort_fields': (  # source_preference is lower for throttled/potentially damaged formats
4300                 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto')
4301         }
4302
4303         subtitles = {}
4304         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
4305         if pctr:
4306             def get_lang_code(track):
4307                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
4308                         or track.get('languageCode'))
4309
4310             # Converted into dicts to remove duplicates
4311             captions = {
4312                 get_lang_code(sub): sub
4313                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...))}
4314             translation_languages = {
4315                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
4316                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...))}
4317
4318             def process_language(container, base_url, lang_code, sub_name, query):
4319                 lang_subs = container.setdefault(lang_code, [])
4320                 for fmt in self._SUBTITLE_FORMATS:
4321                     query.update({
4322                         'fmt': fmt,
4323                     })
4324                     lang_subs.append({
4325                         'ext': fmt,
4326                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
4327                         'name': sub_name,
4328                     })
4329
4330             # NB: Constructing the full subtitle dictionary is slow
4331             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
4332                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
4333             for lang_code, caption_track in captions.items():
4334                 base_url = caption_track.get('baseUrl')
4335                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
4336                 if not base_url:
4337                     continue
4338                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
4339                 if caption_track.get('kind') != 'asr':
4340                     if not lang_code:
4341                         continue
4342                     process_language(
4343                         subtitles, base_url, lang_code, lang_name, {})
4344                     if not caption_track.get('isTranslatable'):
4345                         continue
4346                 for trans_code, trans_name in translation_languages.items():
4347                     if not trans_code:
4348                         continue
4349                     orig_trans_code = trans_code
4350                     if caption_track.get('kind') != 'asr' and trans_code != 'und':
4351                         if not get_translated_subs:
4352                             continue
4353                         trans_code += f'-{lang_code}'
4354                         trans_name += format_field(lang_name, None, ' from %s')
4355                     if lang_code == f'a-{orig_trans_code}':
4356                         # Set audio language based on original subtitles
4357                         for f in formats:
4358                             if f.get('acodec') != 'none' and not f.get('language'):
4359                                 f['language'] = orig_trans_code
4360                         # Add an "-orig" label to the original language so that it can be distinguished.
4361                         # The subs are returned without "-orig" as well for compatibility
4362                         process_language(
4363                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
4364                     # Setting tlang=lang returns damaged subtitles.
4365                     process_language(automatic_captions, base_url, trans_code, trans_name,
4366                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
4367
4368         info['automatic_captions'] = automatic_captions
4369         info['subtitles'] = subtitles
4370
4371         parsed_url = urllib.parse.urlparse(url)
4372         for component in [parsed_url.fragment, parsed_url.query]:
4373             query = urllib.parse.parse_qs(component)
4374             for k, v in query.items():
4375                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4376                     d_k += '_time'
4377                     if d_k not in info and k in s_ks:
4378                         info[d_k] = parse_duration(query[k][0])
4379
4380         # Youtube Music Auto-generated description
4381         if (video_description or '').strip().endswith('\nAuto-generated by YouTube.'):
4382             # XXX: Causes catastrophic backtracking if description has "·"
4383             # E.g. https://www.youtube.com/watch?v=DoPaAxMQoiI
4384             # Simulating atomic groups:  (?P<a>[^xy]+)x  =>  (?=(?P<a>[^xy]+))(?P=a)x
4385             # reduces it, but does not fully fix it. https://regex101.com/r/8Ssf2h/2
4386             mobj = re.search(
4387                 r'''(?xs)
4388                     (?=(?P<track>[^\n·]+))(?P=track)·
4389                     (?=(?P<artist>[^\n]+))(?P=artist)\n+
4390                     (?=(?P<album>[^\n]+))(?P=album)\n
4391                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4392                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4393                     (.+?\nArtist\s*:\s*
4394                         (?=(?P<clean_artist>[^\n]+))(?P=clean_artist)\n
4395                     )?.+\nAuto-generated\ by\ YouTube\.\s*$
4396                 ''', video_description)
4397             if mobj:
4398                 release_year = mobj.group('release_year')
4399                 release_date = mobj.group('release_date')
4400                 if release_date:
4401                     release_date = release_date.replace('-', '')
4402                     if not release_year:
4403                         release_year = release_date[:4]
4404                 info.update({
4405                     'album': mobj.group('album'.strip()),
4406                     'artists': ([a] if (a := mobj.group('clean_artist'))
4407                                 else [a.strip() for a in mobj.group('artist').split('·')]),
4408                     'track': mobj.group('track').strip(),
4409                     'release_date': release_date,
4410                     'release_year': int_or_none(release_year),
4411                 })
4412
4413         initial_data = None
4414         if webpage:
4415             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4416             if not traverse_obj(initial_data, 'contents'):
4417                 self.report_warning('Incomplete data received in embedded initial data; re-fetching using API.')
4418                 initial_data = None
4419         if not initial_data:
4420             query = {'videoId': video_id}
4421             query.update(self._get_checkok_params())
4422             initial_data = self._extract_response(
4423                 item_id=video_id, ep='next', fatal=False,
4424                 ytcfg=master_ytcfg, query=query, check_get_keys='contents',
4425                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4426                 note='Downloading initial data API JSON')
4427
4428         info['comment_count'] = traverse_obj(initial_data, (
4429             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4430             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount'
4431         ), (
4432             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4433             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo'
4434         ), expected_type=self._get_count, get_all=False)
4435
4436         try:  # This will error if there is no livechat
4437             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4438         except (KeyError, IndexError, TypeError):
4439             pass
4440         else:
4441             info.setdefault('subtitles', {})['live_chat'] = [{
4442                 # url is needed to set cookies
4443                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4444                 'video_id': video_id,
4445                 'ext': 'json',
4446                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4447                              else 'youtube_live_chat_replay'),
4448             }]
4449
4450         if initial_data:
4451             info['chapters'] = (
4452                 self._extract_chapters_from_json(initial_data, duration)
4453                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4454                 or self._extract_chapters_from_description(video_description, duration)
4455                 or None)
4456
4457             info['heatmap'] = self._extract_heatmap(initial_data)
4458
4459         contents = traverse_obj(
4460             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4461             expected_type=list, default=[])
4462
4463         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4464         if vpir:
4465             stl = vpir.get('superTitleLink')
4466             if stl:
4467                 stl = self._get_text(stl)
4468                 if try_get(
4469                         vpir,
4470                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4471                     info['location'] = stl
4472                 else:
4473                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4474                     if mobj:
4475                         info.update({
4476                             'series': mobj.group(1),
4477                             'season_number': int(mobj.group(2)),
4478                             'episode_number': int(mobj.group(3)),
4479                         })
4480             for tlb in (try_get(
4481                     vpir,
4482                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4483                     list) or []):
4484                 tbrs = variadic(
4485                     traverse_obj(
4486                         tlb, ('toggleButtonRenderer', ...),
4487                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer')))
4488                 for tbr in tbrs:
4489                     for getter, regex in [(
4490                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4491                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4492                                 lambda x: x['accessibility'],
4493                                 lambda x: x['accessibilityData']['accessibilityData'],
4494                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4495                         label = (try_get(tbr, getter, dict) or {}).get('label')
4496                         if label:
4497                             mobj = re.match(regex, label)
4498                             if mobj:
4499                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4500                                 break
4501
4502             info['like_count'] = traverse_obj(vpir, (
4503                 'videoActions', 'menuRenderer', 'topLevelButtons', ...,
4504                 'segmentedLikeDislikeButtonViewModel', 'likeButtonViewModel', 'likeButtonViewModel',
4505                 'toggleButtonViewModel', 'toggleButtonViewModel', 'defaultButtonViewModel',
4506                 'buttonViewModel', 'accessibilityText', {parse_count}), get_all=False)
4507
4508             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4509             if vcr:
4510                 vc = self._get_count(vcr, 'viewCount')
4511                 # Upcoming premieres with waiting count are treated as live here
4512                 if vcr.get('isLive'):
4513                     info['concurrent_view_count'] = vc
4514                 elif info.get('view_count') is None:
4515                     info['view_count'] = vc
4516
4517         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4518         if vsir:
4519             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4520             info.update({
4521                 'channel': self._get_text(vor, 'title'),
4522                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4523
4524             if not channel_handle:
4525                 channel_handle = self.handle_from_url(
4526                     traverse_obj(vor, (
4527                         ('navigationEndpoint', ('title', 'runs', ..., 'navigationEndpoint')),
4528                         (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl')),
4529                         {str}), get_all=False))
4530
4531             rows = try_get(
4532                 vsir,
4533                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4534                 list) or []
4535             multiple_songs = False
4536             for row in rows:
4537                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4538                     multiple_songs = True
4539                     break
4540             for row in rows:
4541                 mrr = row.get('metadataRowRenderer') or {}
4542                 mrr_title = mrr.get('title')
4543                 if not mrr_title:
4544                     continue
4545                 mrr_title = self._get_text(mrr, 'title')
4546                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4547                 if mrr_title == 'License':
4548                     info['license'] = mrr_contents_text
4549                 elif not multiple_songs:
4550                     if mrr_title == 'Album':
4551                         info['album'] = mrr_contents_text
4552                     elif mrr_title == 'Artist':
4553                         info['artists'] = [mrr_contents_text] if mrr_contents_text else None
4554                     elif mrr_title == 'Song':
4555                         info['track'] = mrr_contents_text
4556             owner_badges = self._extract_badges(traverse_obj(vsir, ('owner', 'videoOwnerRenderer', 'badges')))
4557             if self._has_badge(owner_badges, BadgeType.VERIFIED):
4558                 info['channel_is_verified'] = True
4559
4560         info.update({
4561             'uploader': info.get('channel'),
4562             'uploader_id': channel_handle,
4563             'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
4564         })
4565         # The upload date for scheduled, live and past live streams / premieres in microformats
4566         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4567         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4568         upload_date = (
4569             unified_strdate(get_first(microformats, 'uploadDate'))
4570             or unified_strdate(search_meta('uploadDate')))
4571         if not upload_date or (
4572             live_status in ('not_live', None)
4573             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4574         ):
4575             upload_date = strftime_or_none(
4576                 self._parse_time_text(self._get_text(vpir, 'dateText'))) or upload_date
4577         info['upload_date'] = upload_date
4578
4579         if upload_date and live_status not in ('is_live', 'post_live', 'is_upcoming'):
4580             # Newly uploaded videos' HLS formats are potentially problematic and need to be checked
4581             upload_datetime = datetime_from_str(upload_date).replace(tzinfo=dt.timezone.utc)
4582             if upload_datetime >= datetime_from_str('today-2days'):
4583                 for fmt in info['formats']:
4584                     if fmt.get('protocol') == 'm3u8_native':
4585                         fmt['__needs_testing'] = True
4586
4587         for s_k, d_k in [('artists', 'creators'), ('track', 'alt_title')]:
4588             v = info.get(s_k)
4589             if v:
4590                 info[d_k] = v
4591
4592         badges = self._extract_badges(traverse_obj(vpir, 'badges'))
4593
4594         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4595                       or get_first(video_details, 'isPrivate', expected_type=bool))
4596
4597         info['availability'] = (
4598             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4599             else self._availability(
4600                 is_private=is_private,
4601                 needs_premium=(
4602                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4603                     or False if initial_data and is_private is not None else None),
4604                 needs_subscription=(
4605                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4606                     or False if initial_data and is_private is not None else None),
4607                 needs_auth=info['age_limit'] >= 18,
4608                 is_unlisted=None if is_private is None else (
4609                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4610                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4611
4612         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4613
4614         self.mark_watched(video_id, player_responses)
4615
4616         return info
4617
4618
4619 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4620     @staticmethod
4621     def passthrough_smuggled_data(func):
4622         def _smuggle(info, smuggled_data):
4623             if info.get('_type') not in ('url', 'url_transparent'):
4624                 return info
4625             if smuggled_data.get('is_music_url'):
4626                 parsed_url = urllib.parse.urlparse(info['url'])
4627                 if parsed_url.netloc in ('www.youtube.com', 'music.youtube.com'):
4628                     smuggled_data.pop('is_music_url')
4629                     info['url'] = urllib.parse.urlunparse(parsed_url._replace(netloc='music.youtube.com'))
4630             if smuggled_data:
4631                 info['url'] = smuggle_url(info['url'], smuggled_data)
4632             return info
4633
4634         @functools.wraps(func)
4635         def wrapper(self, url):
4636             url, smuggled_data = unsmuggle_url(url, {})
4637             if self.is_music_url(url):
4638                 smuggled_data['is_music_url'] = True
4639             info_dict = func(self, url, smuggled_data)
4640             if smuggled_data:
4641                 _smuggle(info_dict, smuggled_data)
4642                 if info_dict.get('entries'):
4643                     info_dict['entries'] = (_smuggle(i, smuggled_data.copy()) for i in info_dict['entries'])
4644             return info_dict
4645         return wrapper
4646
4647     @staticmethod
4648     def _extract_basic_item_renderer(item):
4649         # Modified from _extract_grid_item_renderer
4650         known_basic_renderers = (
4651             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4652         )
4653         for key, renderer in item.items():
4654             if not isinstance(renderer, dict):
4655                 continue
4656             elif key in known_basic_renderers:
4657                 return renderer
4658             elif key.startswith('grid') and key.endswith('Renderer'):
4659                 return renderer
4660
4661     def _extract_channel_renderer(self, renderer):
4662         channel_id = self.ucid_or_none(renderer['channelId'])
4663         title = self._get_text(renderer, 'title')
4664         channel_url = format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None)
4665         channel_handle = self.handle_from_url(
4666             traverse_obj(renderer, (
4667                 'navigationEndpoint', (('commandMetadata', 'webCommandMetadata', 'url'),
4668                                        ('browseEndpoint', 'canonicalBaseUrl')),
4669                 {str}), get_all=False))
4670         if not channel_handle:
4671             # As of 2023-06-01, YouTube sets subscriberCountText to the handle in search
4672             channel_handle = self.handle_or_none(self._get_text(renderer, 'subscriberCountText'))
4673         return {
4674             '_type': 'url',
4675             'url': channel_url,
4676             'id': channel_id,
4677             'ie_key': YoutubeTabIE.ie_key(),
4678             'channel': title,
4679             'uploader': title,
4680             'channel_id': channel_id,
4681             'channel_url': channel_url,
4682             'title': title,
4683             'uploader_id': channel_handle,
4684             'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
4685             # See above. YouTube sets videoCountText to the subscriber text in search channel renderers.
4686             # However, in feed/channels this is set correctly to the subscriber count
4687             'channel_follower_count': traverse_obj(
4688                 renderer, 'subscriberCountText', 'videoCountText', expected_type=self._get_count),
4689             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
4690             'playlist_count': (
4691                 # videoCountText may be the subscriber count
4692                 self._get_count(renderer, 'videoCountText')
4693                 if self._get_count(renderer, 'subscriberCountText') is not None else None),
4694             'description': self._get_text(renderer, 'descriptionSnippet'),
4695             'channel_is_verified': True if self._has_badge(
4696                 self._extract_badges(traverse_obj(renderer, 'ownerBadges')), BadgeType.VERIFIED) else None,
4697         }
4698
4699     def _grid_entries(self, grid_renderer):
4700         for item in grid_renderer['items']:
4701             if not isinstance(item, dict):
4702                 continue
4703             renderer = self._extract_basic_item_renderer(item)
4704             if not isinstance(renderer, dict):
4705                 continue
4706             title = self._get_text(renderer, 'title')
4707
4708             # playlist
4709             playlist_id = renderer.get('playlistId')
4710             if playlist_id:
4711                 yield self.url_result(
4712                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4713                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4714                     video_title=title)
4715                 continue
4716             # video
4717             video_id = renderer.get('videoId')
4718             if video_id:
4719                 yield self._extract_video(renderer)
4720                 continue
4721             # channel
4722             channel_id = renderer.get('channelId')
4723             if channel_id:
4724                 yield self._extract_channel_renderer(renderer)
4725                 continue
4726             # generic endpoint URL support
4727             ep_url = urljoin('https://www.youtube.com/', try_get(
4728                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4729                 str))
4730             if ep_url:
4731                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4732                     if ie.suitable(ep_url):
4733                         yield self.url_result(
4734                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4735                         break
4736
4737     def _music_reponsive_list_entry(self, renderer):
4738         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4739         if video_id:
4740             title = traverse_obj(renderer, (
4741                 'flexColumns', 0, 'musicResponsiveListItemFlexColumnRenderer',
4742                 'text', 'runs', 0, 'text'))
4743             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4744                                    ie=YoutubeIE.ie_key(), video_id=video_id, title=title)
4745         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4746         if playlist_id:
4747             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4748             if video_id:
4749                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4750                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4751             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4752                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4753         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4754         if browse_id:
4755             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4756                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4757
4758     def _shelf_entries_from_content(self, shelf_renderer):
4759         content = shelf_renderer.get('content')
4760         if not isinstance(content, dict):
4761             return
4762         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4763         if renderer:
4764             # TODO: add support for nested playlists so each shelf is processed
4765             # as separate playlist
4766             # TODO: this includes only first N items
4767             yield from self._grid_entries(renderer)
4768         renderer = content.get('horizontalListRenderer')
4769         if renderer:
4770             # TODO
4771             pass
4772
4773     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4774         ep = try_get(
4775             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4776             str)
4777         shelf_url = urljoin('https://www.youtube.com', ep)
4778         if shelf_url:
4779             # Skipping links to another channels, note that checking for
4780             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4781             # will not work
4782             if skip_channels and '/channels?' in shelf_url:
4783                 return
4784             title = self._get_text(shelf_renderer, 'title')
4785             yield self.url_result(shelf_url, video_title=title)
4786         # Shelf may not contain shelf URL, fallback to extraction from content
4787         yield from self._shelf_entries_from_content(shelf_renderer)
4788
4789     def _playlist_entries(self, video_list_renderer):
4790         for content in video_list_renderer['contents']:
4791             if not isinstance(content, dict):
4792                 continue
4793             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4794             if not isinstance(renderer, dict):
4795                 continue
4796             video_id = renderer.get('videoId')
4797             if not video_id:
4798                 continue
4799             yield self._extract_video(renderer)
4800
4801     def _rich_entries(self, rich_grid_renderer):
4802         renderer = traverse_obj(
4803             rich_grid_renderer,
4804             ('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer')), get_all=False) or {}
4805         video_id = renderer.get('videoId')
4806         if video_id:
4807             yield self._extract_video(renderer)
4808             return
4809         playlist_id = renderer.get('playlistId')
4810         if playlist_id:
4811             yield self.url_result(
4812                 f'https://www.youtube.com/playlist?list={playlist_id}',
4813                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4814                 video_title=self._get_text(renderer, 'title'))
4815             return
4816
4817     def _video_entry(self, video_renderer):
4818         video_id = video_renderer.get('videoId')
4819         if video_id:
4820             return self._extract_video(video_renderer)
4821
4822     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4823         url = urljoin('https://youtube.com', traverse_obj(
4824             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4825         if url:
4826             return self.url_result(
4827                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4828
4829     def _post_thread_entries(self, post_thread_renderer):
4830         post_renderer = try_get(
4831             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4832         if not post_renderer:
4833             return
4834         # video attachment
4835         video_renderer = try_get(
4836             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4837         video_id = video_renderer.get('videoId')
4838         if video_id:
4839             entry = self._extract_video(video_renderer)
4840             if entry:
4841                 yield entry
4842         # playlist attachment
4843         playlist_id = try_get(
4844             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4845         if playlist_id:
4846             yield self.url_result(
4847                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4848                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4849         # inline video links
4850         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4851         for run in runs:
4852             if not isinstance(run, dict):
4853                 continue
4854             ep_url = try_get(
4855                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4856             if not ep_url:
4857                 continue
4858             if not YoutubeIE.suitable(ep_url):
4859                 continue
4860             ep_video_id = YoutubeIE._match_id(ep_url)
4861             if video_id == ep_video_id:
4862                 continue
4863             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4864
4865     def _post_thread_continuation_entries(self, post_thread_continuation):
4866         contents = post_thread_continuation.get('contents')
4867         if not isinstance(contents, list):
4868             return
4869         for content in contents:
4870             renderer = content.get('backstagePostThreadRenderer')
4871             if isinstance(renderer, dict):
4872                 yield from self._post_thread_entries(renderer)
4873                 continue
4874             renderer = content.get('videoRenderer')
4875             if isinstance(renderer, dict):
4876                 yield self._video_entry(renderer)
4877
4878     r''' # unused
4879     def _rich_grid_entries(self, contents):
4880         for content in contents:
4881             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4882             if video_renderer:
4883                 entry = self._video_entry(video_renderer)
4884                 if entry:
4885                     yield entry
4886     '''
4887
4888     def _report_history_entries(self, renderer):
4889         for url in traverse_obj(renderer, (
4890                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4891                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4892                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4893             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4894
4895     def _extract_entries(self, parent_renderer, continuation_list):
4896         # continuation_list is modified in-place with continuation_list = [continuation_token]
4897         continuation_list[:] = [None]
4898         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4899         for content in contents:
4900             if not isinstance(content, dict):
4901                 continue
4902             is_renderer = traverse_obj(
4903                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4904                 expected_type=dict)
4905             if not is_renderer:
4906                 if content.get('richItemRenderer'):
4907                     for entry in self._rich_entries(content['richItemRenderer']):
4908                         yield entry
4909                     continuation_list[0] = self._extract_continuation(parent_renderer)
4910                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4911                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4912                     yield from self._report_history_entries(table)
4913                     continuation_list[0] = self._extract_continuation(table)
4914                 continue
4915
4916             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4917             for isr_content in isr_contents:
4918                 if not isinstance(isr_content, dict):
4919                     continue
4920
4921                 known_renderers = {
4922                     'playlistVideoListRenderer': self._playlist_entries,
4923                     'gridRenderer': self._grid_entries,
4924                     'reelShelfRenderer': self._grid_entries,
4925                     'shelfRenderer': self._shelf_entries,
4926                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4927                     'backstagePostThreadRenderer': self._post_thread_entries,
4928                     'videoRenderer': lambda x: [self._video_entry(x)],
4929                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4930                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4931                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)],
4932                     'richGridRenderer': lambda x: self._extract_entries(x, continuation_list),
4933                 }
4934                 for key, renderer in isr_content.items():
4935                     if key not in known_renderers:
4936                         continue
4937                     for entry in known_renderers[key](renderer):
4938                         if entry:
4939                             yield entry
4940                     continuation_list[0] = self._extract_continuation(renderer)
4941                     break
4942
4943             if not continuation_list[0]:
4944                 continuation_list[0] = self._extract_continuation(is_renderer)
4945
4946         if not continuation_list[0]:
4947             continuation_list[0] = self._extract_continuation(parent_renderer)
4948
4949     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4950         continuation_list = [None]
4951         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4952         tab_content = try_get(tab, lambda x: x['content'], dict)
4953         if not tab_content:
4954             return
4955         parent_renderer = (
4956             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4957             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4958         yield from extract_entries(parent_renderer)
4959         continuation = continuation_list[0]
4960         seen_continuations = set()
4961         for page_num in itertools.count(1):
4962             if not continuation:
4963                 break
4964             continuation_token = continuation.get('continuation')
4965             if continuation_token is not None and continuation_token in seen_continuations:
4966                 self.write_debug('Detected YouTube feed looping - assuming end of feed.')
4967                 break
4968             seen_continuations.add(continuation_token)
4969             headers = self.generate_api_headers(
4970                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4971             response = self._extract_response(
4972                 item_id=f'{item_id} page {page_num}',
4973                 query=continuation, headers=headers, ytcfg=ytcfg,
4974                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4975
4976             if not response:
4977                 break
4978             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4979             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4980             visitor_data = self._extract_visitor_data(response) or visitor_data
4981
4982             known_renderers = {
4983                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4984                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4985                 'gridVideoRenderer': (self._grid_entries, 'items'),
4986                 'gridChannelRenderer': (self._grid_entries, 'items'),
4987                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4988                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4989                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4990                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4991                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4992                 'playlistVideoListContinuation': (self._playlist_entries, None),
4993                 'gridContinuation': (self._grid_entries, None),
4994                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4995                 'sectionListContinuation': (extract_entries, None),  # for feeds
4996             }
4997
4998             continuation_items = traverse_obj(response, (
4999                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
5000                 'appendContinuationItemsAction', 'continuationItems'
5001             ), 'continuationContents', get_all=False)
5002             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
5003
5004             video_items_renderer = None
5005             for key in continuation_item.keys():
5006                 if key not in known_renderers:
5007                     continue
5008                 func, parent_key = known_renderers[key]
5009                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
5010                 continuation_list = [None]
5011                 yield from func(video_items_renderer)
5012                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
5013
5014             if not video_items_renderer:
5015                 break
5016
5017     @staticmethod
5018     def _extract_selected_tab(tabs, fatal=True):
5019         for tab_renderer in tabs:
5020             if tab_renderer.get('selected'):
5021                 return tab_renderer
5022         if fatal:
5023             raise ExtractorError('Unable to find selected tab')
5024
5025     @staticmethod
5026     def _extract_tab_renderers(response):
5027         return traverse_obj(
5028             response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
5029
5030     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
5031         metadata = self._extract_metadata_from_tabs(item_id, data)
5032
5033         selected_tab = self._extract_selected_tab(tabs)
5034         metadata['title'] += format_field(selected_tab, 'title', ' - %s')
5035         metadata['title'] += format_field(selected_tab, 'expandedText', ' - %s')
5036
5037         return self.playlist_result(
5038             self._entries(
5039                 selected_tab, metadata['id'], ytcfg,
5040                 self._extract_account_syncid(ytcfg, data),
5041                 self._extract_visitor_data(data, ytcfg)),
5042             **metadata)
5043
5044     def _extract_metadata_from_tabs(self, item_id, data):
5045         info = {'id': item_id}
5046
5047         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
5048         if metadata_renderer:
5049             channel_id = traverse_obj(metadata_renderer, ('externalId', {self.ucid_or_none}),
5050                                       ('channelUrl', {self.ucid_from_url}))
5051             info.update({
5052                 'channel': metadata_renderer.get('title'),
5053                 'channel_id': channel_id,
5054             })
5055             if info['channel_id']:
5056                 info['id'] = info['channel_id']
5057         else:
5058             metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
5059
5060         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
5061         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
5062         def _get_uncropped(url):
5063             return url_or_none((url or '').split('=')[0] + '=s0')
5064
5065         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
5066         if avatar_thumbnails:
5067             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
5068             if uncropped_avatar:
5069                 avatar_thumbnails.append({
5070                     'url': uncropped_avatar,
5071                     'id': 'avatar_uncropped',
5072                     'preference': 1
5073                 })
5074
5075         channel_banners = self._extract_thumbnails(
5076             data, ('header', ..., ('banner', 'mobileBanner', 'tvBanner')))
5077         for banner in channel_banners:
5078             banner['preference'] = -10
5079
5080         if channel_banners:
5081             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
5082             if uncropped_banner:
5083                 channel_banners.append({
5084                     'url': uncropped_banner,
5085                     'id': 'banner_uncropped',
5086                     'preference': -5
5087                 })
5088
5089         # Deprecated - remove primary_sidebar_renderer when layout discontinued
5090         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
5091         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
5092
5093         primary_thumbnails = self._extract_thumbnails(
5094             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
5095         playlist_thumbnails = self._extract_thumbnails(
5096             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
5097
5098         info.update({
5099             'title': (traverse_obj(metadata_renderer, 'title')
5100                       or self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag'))
5101                       or info['id']),
5102             'availability': self._extract_availability(data),
5103             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
5104             'description': try_get(metadata_renderer, lambda x: x.get('description', '')),
5105             'tags': (traverse_obj(data, ('microformat', 'microformatDataRenderer', 'tags', ..., {str}))
5106                      or traverse_obj(metadata_renderer, ('keywords', {lambda x: x and shlex.split(x)}, ...))),
5107             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
5108         })
5109
5110         channel_handle = (
5111             traverse_obj(metadata_renderer, (('vanityChannelUrl', ('ownerUrls', ...)), {self.handle_from_url}), get_all=False)
5112             or traverse_obj(data, ('header', ..., 'channelHandleText', {self.handle_or_none}), get_all=False))
5113
5114         if channel_handle:
5115             info.update({
5116                 'uploader_id': channel_handle,
5117                 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
5118             })
5119
5120         channel_badges = self._extract_badges(traverse_obj(data, ('header', ..., 'badges'), get_all=False))
5121         if self._has_badge(channel_badges, BadgeType.VERIFIED):
5122             info['channel_is_verified'] = True
5123         # Playlist stats is a text runs array containing [video count, view count, last updated].
5124         # last updated or (view count and last updated) may be missing.
5125         playlist_stats = get_first(
5126             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'), ))
5127
5128         last_updated_unix = self._parse_time_text(
5129             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
5130             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
5131         info['modified_date'] = strftime_or_none(last_updated_unix)
5132
5133         info['view_count'] = self._get_count(playlist_stats, 1)
5134         if info['view_count'] is None:  # 0 is allowed
5135             info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
5136         if info['view_count'] is None:
5137             info['view_count'] = self._get_count(data, (
5138                 'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer',
5139                 'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText'))
5140
5141         info['playlist_count'] = self._get_count(playlist_stats, 0)
5142         if info['playlist_count'] is None:  # 0 is allowed
5143             info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
5144
5145         if not info.get('channel_id'):
5146             owner = traverse_obj(playlist_header_renderer, 'ownerText')
5147             if not owner:  # Deprecated
5148                 owner = traverse_obj(
5149                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
5150                     ('videoOwner', 'videoOwnerRenderer', 'title'))
5151             owner_text = self._get_text(owner)
5152             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
5153             info.update({
5154                 'channel': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
5155                 'channel_id': self.ucid_or_none(browse_ep.get('browseId')),
5156                 'uploader_id': self.handle_from_url(urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl')))
5157             })
5158
5159         info.update({
5160             'uploader': info['channel'],
5161             'channel_url': format_field(info.get('channel_id'), None, 'https://www.youtube.com/channel/%s', default=None),
5162             'uploader_url': format_field(info.get('uploader_id'), None, 'https://www.youtube.com/%s', default=None),
5163         })
5164
5165         return info
5166
5167     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
5168         first_id = last_id = response = None
5169         for page_num in itertools.count(1):
5170             videos = list(self._playlist_entries(playlist))
5171             if not videos:
5172                 return
5173             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
5174             if start >= len(videos):
5175                 return
5176             yield from videos[start:]
5177             first_id = first_id or videos[0]['id']
5178             last_id = videos[-1]['id']
5179             watch_endpoint = try_get(
5180                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
5181             headers = self.generate_api_headers(
5182                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
5183                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
5184             query = {
5185                 'playlistId': playlist_id,
5186                 'videoId': watch_endpoint.get('videoId') or last_id,
5187                 'index': watch_endpoint.get('index') or len(videos),
5188                 'params': watch_endpoint.get('params') or 'OAE%3D'
5189             }
5190             response = self._extract_response(
5191                 item_id='%s page %d' % (playlist_id, page_num),
5192                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
5193                 check_get_keys='contents'
5194             )
5195             playlist = try_get(
5196                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
5197
5198     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
5199         title = playlist.get('title') or try_get(
5200             data, lambda x: x['titleText']['simpleText'], str)
5201         playlist_id = playlist.get('playlistId') or item_id
5202
5203         # Delegating everything except mix playlists to regular tab-based playlist URL
5204         playlist_url = urljoin(url, try_get(
5205             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
5206             str))
5207
5208         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
5209         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
5210         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
5211
5212         if playlist_url and playlist_url != url and not is_known_unviewable:
5213             return self.url_result(
5214                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
5215                 video_title=title)
5216
5217         return self.playlist_result(
5218             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
5219             playlist_id=playlist_id, playlist_title=title)
5220
5221     def _extract_availability(self, data):
5222         """
5223         Gets the availability of a given playlist/tab.
5224         Note: Unless YouTube tells us explicitly, we do not assume it is public
5225         @param data: response
5226         """
5227         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
5228         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
5229         player_header_privacy = playlist_header_renderer.get('privacy')
5230
5231         badges = self._extract_badges(traverse_obj(sidebar_renderer, 'badges'))
5232
5233         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
5234         privacy_setting_icon = get_first(
5235             (playlist_header_renderer, sidebar_renderer),
5236             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
5237              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
5238             expected_type=str)
5239
5240         microformats_is_unlisted = traverse_obj(
5241             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
5242
5243         return (
5244             'public' if (
5245                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
5246                 or player_header_privacy == 'PUBLIC'
5247                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
5248             else self._availability(
5249                 is_private=(
5250                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
5251                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
5252                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
5253                 is_unlisted=(
5254                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
5255                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
5256                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
5257                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
5258                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
5259                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
5260                 needs_auth=False))
5261
5262     @staticmethod
5263     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
5264         sidebar_renderer = try_get(
5265             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
5266         for item in sidebar_renderer:
5267             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
5268             if renderer:
5269                 return renderer
5270
5271     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
5272         """
5273         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
5274         """
5275         is_playlist = bool(traverse_obj(
5276             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
5277         if not is_playlist:
5278             return
5279         headers = self.generate_api_headers(
5280             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
5281             visitor_data=self._extract_visitor_data(data, ytcfg))
5282         query = {
5283             'params': 'wgYCCAA=',
5284             'browseId': f'VL{item_id}'
5285         }
5286         return self._extract_response(
5287             item_id=item_id, headers=headers, query=query,
5288             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
5289             note='Redownloading playlist API JSON with unavailable videos')
5290
5291     @functools.cached_property
5292     def skip_webpage(self):
5293         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
5294
5295     def _extract_webpage(self, url, item_id, fatal=True):
5296         webpage, data = None, None
5297         for retry in self.RetryManager(fatal=fatal):
5298             try:
5299                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
5300                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
5301             except ExtractorError as e:
5302                 if isinstance(e.cause, network_exceptions):
5303                     if not isinstance(e.cause, HTTPError) or e.cause.status not in (403, 429):
5304                         retry.error = e
5305                         continue
5306                 self._error_or_warning(e, fatal=fatal)
5307                 break
5308
5309             try:
5310                 self._extract_and_report_alerts(data)
5311             except ExtractorError as e:
5312                 self._error_or_warning(e, fatal=fatal)
5313                 break
5314
5315             # Sometimes youtube returns a webpage with incomplete ytInitialData
5316             # See: https://github.com/yt-dlp/yt-dlp/issues/116
5317             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
5318                 retry.error = ExtractorError('Incomplete yt initial data received')
5319                 data = None
5320                 continue
5321
5322         return webpage, data
5323
5324     def _report_playlist_authcheck(self, ytcfg, fatal=True):
5325         """Use if failed to extract ytcfg (and data) from initial webpage"""
5326         if not ytcfg and self.is_authenticated:
5327             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
5328             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
5329                 raise ExtractorError(
5330                     f'{msg}. If you are not downloading private content, or '
5331                     'your cookies are only for the first account and channel,'
5332                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
5333                     expected=True)
5334             self.report_warning(msg, only_once=True)
5335
5336     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
5337         data = None
5338         if not self.skip_webpage:
5339             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
5340             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
5341             # Reject webpage data if redirected to home page without explicitly requesting
5342             selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
5343             if (url != 'https://www.youtube.com/feed/recommended'
5344                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
5345                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
5346                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
5347                 if fatal:
5348                     raise ExtractorError(msg, expected=True)
5349                 self.report_warning(msg, only_once=True)
5350         if not data:
5351             self._report_playlist_authcheck(ytcfg, fatal=fatal)
5352             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
5353         return data, ytcfg
5354
5355     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
5356         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
5357         resolve_response = self._extract_response(
5358             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
5359             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
5360         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
5361         for ep_key, ep in endpoints.items():
5362             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
5363             if params:
5364                 return self._extract_response(
5365                     item_id=item_id, query=params, ep=ep, headers=headers,
5366                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
5367                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
5368         err_note = 'Failed to resolve url (does the playlist exist?)'
5369         if fatal:
5370             raise ExtractorError(err_note, expected=True)
5371         self.report_warning(err_note, item_id)
5372
5373     _SEARCH_PARAMS = None
5374
5375     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
5376         data = {'query': query}
5377         if params is NO_DEFAULT:
5378             params = self._SEARCH_PARAMS
5379         if params:
5380             data['params'] = params
5381
5382         content_keys = (
5383             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
5384             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
5385             # ytmusic search
5386             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
5387             ('continuationContents', ),
5388         )
5389         display_id = f'query "{query}"'
5390         check_get_keys = tuple({keys[0] for keys in content_keys})
5391         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
5392         self._report_playlist_authcheck(ytcfg, fatal=False)
5393
5394         continuation_list = [None]
5395         search = None
5396         for page_num in itertools.count(1):
5397             data.update(continuation_list[0] or {})
5398             headers = self.generate_api_headers(
5399                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
5400             search = self._extract_response(
5401                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
5402                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
5403             slr_contents = traverse_obj(search, *content_keys)
5404             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
5405             if not continuation_list[0]:
5406                 break
5407
5408
5409 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
5410     IE_DESC = 'YouTube Tabs'
5411     _VALID_URL = r'''(?x:
5412         https?://
5413             (?!consent\.)(?:\w+\.)?
5414             (?:
5415                 youtube(?:kids)?\.com|
5416                 %(invidious)s
5417             )/
5418             (?:
5419                 (?P<channel_type>channel|c|user|browse)/|
5420                 (?P<not_channel>
5421                     feed/|hashtag/|
5422                     (?:playlist|watch)\?.*?\blist=
5423                 )|
5424                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
5425             )
5426             (?P<id>[^/?\#&]+)
5427     )''' % {
5428         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
5429         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5430     }
5431     IE_NAME = 'youtube:tab'
5432
5433     _TESTS = [{
5434         'note': 'playlists, multipage',
5435         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
5436         'playlist_mincount': 94,
5437         'info_dict': {
5438             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5439             'title': 'Igor Kleiner Ph.D. - Playlists',
5440             'description': 'md5:15d7dd9e333cb987907fcb0d604b233a',
5441             'uploader': 'Igor Kleiner Ph.D.',
5442             'uploader_id': '@IgorDataScience',
5443             'uploader_url': 'https://www.youtube.com/@IgorDataScience',
5444             'channel': 'Igor Kleiner Ph.D.',
5445             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5446             'tags': ['критическое мышление', 'наука просто', 'математика', 'анализ данных'],
5447             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5448             'channel_follower_count': int
5449         },
5450     }, {
5451         'note': 'playlists, multipage, different order',
5452         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5453         'playlist_mincount': 94,
5454         'info_dict': {
5455             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5456             'title': 'Igor Kleiner Ph.D. - Playlists',
5457             'description': 'md5:15d7dd9e333cb987907fcb0d604b233a',
5458             'uploader': 'Igor Kleiner Ph.D.',
5459             'uploader_id': '@IgorDataScience',
5460             'uploader_url': 'https://www.youtube.com/@IgorDataScience',
5461             'tags': ['критическое мышление', 'наука просто', 'математика', 'анализ данных'],
5462             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5463             'channel': 'Igor Kleiner Ph.D.',
5464             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5465             'channel_follower_count': int
5466         },
5467     }, {
5468         'note': 'playlists, series',
5469         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5470         'playlist_mincount': 5,
5471         'info_dict': {
5472             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5473             'title': '3Blue1Brown - Playlists',
5474             'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
5475             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5476             'channel': '3Blue1Brown',
5477             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5478             'uploader_id': '@3blue1brown',
5479             'uploader_url': 'https://www.youtube.com/@3blue1brown',
5480             'uploader': '3Blue1Brown',
5481             'tags': ['Mathematics'],
5482             'channel_follower_count': int,
5483             'channel_is_verified': True,
5484         },
5485     }, {
5486         'note': 'playlists, singlepage',
5487         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5488         'playlist_mincount': 4,
5489         'info_dict': {
5490             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5491             'title': 'ThirstForScience - Playlists',
5492             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5493             'uploader': 'ThirstForScience',
5494             'uploader_url': 'https://www.youtube.com/@ThirstForScience',
5495             'uploader_id': '@ThirstForScience',
5496             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5497             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5498             'tags': 'count:12',
5499             'channel': 'ThirstForScience',
5500             'channel_follower_count': int
5501         }
5502     }, {
5503         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5504         'only_matching': True,
5505     }, {
5506         'note': 'basic, single video playlist',
5507         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5508         'info_dict': {
5509             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5510             'title': 'youtube-dl public playlist',
5511             'description': '',
5512             'tags': [],
5513             'view_count': int,
5514             'modified_date': '20201130',
5515             'channel': 'Sergey M.',
5516             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5517             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5518             'availability': 'public',
5519             'uploader': 'Sergey M.',
5520             'uploader_url': 'https://www.youtube.com/@sergeym.6173',
5521             'uploader_id': '@sergeym.6173',
5522         },
5523         'playlist_count': 1,
5524     }, {
5525         'note': 'empty playlist',
5526         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5527         'info_dict': {
5528             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5529             'title': 'youtube-dl empty playlist',
5530             'tags': [],
5531             'channel': 'Sergey M.',
5532             'description': '',
5533             'modified_date': '20230921',
5534             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5535             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5536             'availability': 'unlisted',
5537             'uploader_url': 'https://www.youtube.com/@sergeym.6173',
5538             'uploader_id': '@sergeym.6173',
5539             'uploader': 'Sergey M.',
5540         },
5541         'playlist_count': 0,
5542     }, {
5543         'note': 'Home tab',
5544         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5545         'info_dict': {
5546             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5547             'title': 'lex will - Home',
5548             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5549             'uploader': 'lex will',
5550             'uploader_id': '@lexwill718',
5551             'channel': 'lex will',
5552             'tags': ['bible', 'history', 'prophesy'],
5553             'uploader_url': 'https://www.youtube.com/@lexwill718',
5554             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5555             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5556             'channel_follower_count': int
5557         },
5558         'playlist_mincount': 2,
5559     }, {
5560         'note': 'Videos tab',
5561         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5562         'info_dict': {
5563             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5564             'title': 'lex will - Videos',
5565             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5566             'uploader': 'lex will',
5567             'uploader_id': '@lexwill718',
5568             'tags': ['bible', 'history', 'prophesy'],
5569             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5570             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5571             'uploader_url': 'https://www.youtube.com/@lexwill718',
5572             'channel': 'lex will',
5573             'channel_follower_count': int
5574         },
5575         'playlist_mincount': 975,
5576     }, {
5577         'note': 'Videos tab, sorted by popular',
5578         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5579         'info_dict': {
5580             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5581             'title': 'lex will - Videos',
5582             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5583             'uploader': 'lex will',
5584             'uploader_id': '@lexwill718',
5585             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5586             'uploader_url': 'https://www.youtube.com/@lexwill718',
5587             'channel': 'lex will',
5588             'tags': ['bible', 'history', 'prophesy'],
5589             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5590             'channel_follower_count': int
5591         },
5592         'playlist_mincount': 199,
5593     }, {
5594         'note': 'Playlists tab',
5595         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5596         'info_dict': {
5597             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5598             'title': 'lex will - Playlists',
5599             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5600             'uploader': 'lex will',
5601             'uploader_id': '@lexwill718',
5602             'uploader_url': 'https://www.youtube.com/@lexwill718',
5603             'channel': 'lex will',
5604             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5605             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5606             'tags': ['bible', 'history', 'prophesy'],
5607             'channel_follower_count': int
5608         },
5609         'playlist_mincount': 17,
5610     }, {
5611         'note': 'Community tab',
5612         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5613         'info_dict': {
5614             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5615             'title': 'lex will - Community',
5616             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5617             'channel': 'lex will',
5618             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5619             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5620             'tags': ['bible', 'history', 'prophesy'],
5621             'channel_follower_count': int,
5622             'uploader_url': 'https://www.youtube.com/@lexwill718',
5623             'uploader_id': '@lexwill718',
5624             'uploader': 'lex will',
5625         },
5626         'playlist_mincount': 18,
5627     }, {
5628         'note': 'Channels tab',
5629         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5630         'info_dict': {
5631             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5632             'title': 'lex will - Channels',
5633             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5634             'channel': 'lex will',
5635             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5636             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5637             'tags': ['bible', 'history', 'prophesy'],
5638             'channel_follower_count': int,
5639             'uploader_url': 'https://www.youtube.com/@lexwill718',
5640             'uploader_id': '@lexwill718',
5641             'uploader': 'lex will',
5642         },
5643         'playlist_mincount': 12,
5644     }, {
5645         'note': 'Search tab',
5646         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5647         'playlist_mincount': 40,
5648         'info_dict': {
5649             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5650             'title': '3Blue1Brown - Search - linear algebra',
5651             'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
5652             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5653             'tags': ['Mathematics'],
5654             'channel': '3Blue1Brown',
5655             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5656             'channel_follower_count': int,
5657             'uploader_url': 'https://www.youtube.com/@3blue1brown',
5658             'uploader_id': '@3blue1brown',
5659             'uploader': '3Blue1Brown',
5660             'channel_is_verified': True,
5661         },
5662     }, {
5663         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5664         'only_matching': True,
5665     }, {
5666         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5667         'only_matching': True,
5668     }, {
5669         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5670         'only_matching': True,
5671     }, {
5672         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5673         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5674         'info_dict': {
5675             'title': '29C3: Not my department',
5676             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5677             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5678             'tags': [],
5679             'view_count': int,
5680             'modified_date': '20150605',
5681             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5682             'channel_url': 'https://www.youtube.com/channel/UCEPzS1rYsrkqzSLNp76nrcg',
5683             'channel': 'Christiaan008',
5684             'availability': 'public',
5685             'uploader_id': '@ChRiStIaAn008',
5686             'uploader': 'Christiaan008',
5687             'uploader_url': 'https://www.youtube.com/@ChRiStIaAn008',
5688         },
5689         'playlist_count': 96,
5690     }, {
5691         'note': 'Large playlist',
5692         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5693         'info_dict': {
5694             'title': 'Uploads from Cauchemar',
5695             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5696             'channel_url': 'https://www.youtube.com/channel/UCBABnxM4Ar9ten8Mdjj1j0Q',
5697             'tags': [],
5698             'modified_date': r're:\d{8}',
5699             'channel': 'Cauchemar',
5700             'view_count': int,
5701             'description': '',
5702             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5703             'availability': 'public',
5704             'uploader_id': '@Cauchemar89',
5705             'uploader': 'Cauchemar',
5706             'uploader_url': 'https://www.youtube.com/@Cauchemar89',
5707         },
5708         'playlist_mincount': 1123,
5709         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5710     }, {
5711         'note': 'even larger playlist, 8832 videos',
5712         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5713         'only_matching': True,
5714     }, {
5715         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5716         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5717         'info_dict': {
5718             'title': 'Uploads from Interstellar Movie',
5719             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5720             'tags': [],
5721             'view_count': int,
5722             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5723             'channel_url': 'https://www.youtube.com/channel/UCXw-G3eDE9trcvY2sBMM_aA',
5724             'channel': 'Interstellar Movie',
5725             'description': '',
5726             'modified_date': r're:\d{8}',
5727             'availability': 'public',
5728             'uploader_id': '@InterstellarMovie',
5729             'uploader': 'Interstellar Movie',
5730             'uploader_url': 'https://www.youtube.com/@InterstellarMovie',
5731         },
5732         'playlist_mincount': 21,
5733     }, {
5734         'note': 'Playlist with "show unavailable videos" button',
5735         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5736         'info_dict': {
5737             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5738             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5739             'view_count': int,
5740             'channel': 'Phim Siêu Nhân Nhật Bản',
5741             'tags': [],
5742             'description': '',
5743             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5744             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5745             'modified_date': r're:\d{8}',
5746             'availability': 'public',
5747             'uploader_url': 'https://www.youtube.com/@phimsieunhannhatban',
5748             'uploader_id': '@phimsieunhannhatban',
5749             'uploader': 'Phim Siêu Nhân Nhật Bản',
5750         },
5751         'playlist_mincount': 200,
5752         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5753     }, {
5754         'note': 'Playlist with unavailable videos in page 7',
5755         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5756         'info_dict': {
5757             'title': 'Uploads from BlankTV',
5758             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5759             'channel': 'BlankTV',
5760             'channel_url': 'https://www.youtube.com/channel/UC8l9frL61Yl5KFOl87nIm2w',
5761             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5762             'view_count': int,
5763             'tags': [],
5764             'modified_date': r're:\d{8}',
5765             'description': '',
5766             'availability': 'public',
5767             'uploader_id': '@blanktv',
5768             'uploader': 'BlankTV',
5769             'uploader_url': 'https://www.youtube.com/@blanktv',
5770         },
5771         'playlist_mincount': 1000,
5772         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5773     }, {
5774         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5775         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5776         'info_dict': {
5777             'title': 'Data Analysis with Dr Mike Pound',
5778             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5779             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5780             'tags': [],
5781             'view_count': int,
5782             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5783             'channel_url': 'https://www.youtube.com/channel/UC9-y-6csu5WGm29I7JiwpnA',
5784             'channel': 'Computerphile',
5785             'availability': 'public',
5786             'modified_date': '20190712',
5787             'uploader_id': '@Computerphile',
5788             'uploader': 'Computerphile',
5789             'uploader_url': 'https://www.youtube.com/@Computerphile',
5790         },
5791         'playlist_mincount': 11,
5792     }, {
5793         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5794         'only_matching': True,
5795     }, {
5796         'note': 'Playlist URL that does not actually serve a playlist',
5797         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5798         'info_dict': {
5799             'id': 'FqZTN594JQw',
5800             'ext': 'webm',
5801             'title': "Smiley's People 01 detective, Adventure Series, Action",
5802             'upload_date': '20150526',
5803             'license': 'Standard YouTube License',
5804             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5805             'categories': ['People & Blogs'],
5806             'tags': list,
5807             'view_count': int,
5808             'like_count': int,
5809         },
5810         'params': {
5811             'skip_download': True,
5812         },
5813         'skip': 'This video is not available.',
5814         'add_ie': [YoutubeIE.ie_key()],
5815     }, {
5816         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5817         'only_matching': True,
5818     }, {
5819         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5820         'only_matching': True,
5821     }, {
5822         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5823         'info_dict': {
5824             'id': 'hGkQjiJLjWQ',  # This will keep changing
5825             'ext': 'mp4',
5826             'title': str,
5827             'upload_date': r're:\d{8}',
5828             'description': str,
5829             'categories': ['News & Politics'],
5830             'tags': list,
5831             'like_count': int,
5832             'release_timestamp': int,
5833             'channel': 'Sky News',
5834             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5835             'age_limit': 0,
5836             'view_count': int,
5837             'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
5838             'playable_in_embed': True,
5839             'release_date': r're:\d+',
5840             'availability': 'public',
5841             'live_status': 'is_live',
5842             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5843             'channel_follower_count': int,
5844             'concurrent_view_count': int,
5845             'uploader_url': 'https://www.youtube.com/@SkyNews',
5846             'uploader_id': '@SkyNews',
5847             'uploader': 'Sky News',
5848             'channel_is_verified': True,
5849         },
5850         'params': {
5851             'skip_download': True,
5852         },
5853         'expected_warnings': ['Ignoring subtitle tracks found in '],
5854     }, {
5855         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5856         'info_dict': {
5857             'id': 'a48o2S1cPoo',
5858             'ext': 'mp4',
5859             'title': 'The Young Turks - Live Main Show',
5860             'upload_date': '20150715',
5861             'license': 'Standard YouTube License',
5862             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5863             'categories': ['News & Politics'],
5864             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5865             'like_count': int,
5866         },
5867         'params': {
5868             'skip_download': True,
5869         },
5870         'only_matching': True,
5871     }, {
5872         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5873         'only_matching': True,
5874     }, {
5875         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5876         'only_matching': True,
5877     }, {
5878         'note': 'A channel that is not live. Should raise error',
5879         'url': 'https://www.youtube.com/user/numberphile/live',
5880         'only_matching': True,
5881     }, {
5882         'url': 'https://www.youtube.com/feed/trending',
5883         'only_matching': True,
5884     }, {
5885         'url': 'https://www.youtube.com/feed/library',
5886         'only_matching': True,
5887     }, {
5888         'url': 'https://www.youtube.com/feed/history',
5889         'only_matching': True,
5890     }, {
5891         'url': 'https://www.youtube.com/feed/subscriptions',
5892         'only_matching': True,
5893     }, {
5894         'url': 'https://www.youtube.com/feed/watch_later',
5895         'only_matching': True,
5896     }, {
5897         'note': 'Recommended - redirects to home page.',
5898         'url': 'https://www.youtube.com/feed/recommended',
5899         'only_matching': True,
5900     }, {
5901         'note': 'inline playlist with not always working continuations',
5902         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5903         'only_matching': True,
5904     }, {
5905         'url': 'https://www.youtube.com/course',
5906         'only_matching': True,
5907     }, {
5908         'url': 'https://www.youtube.com/zsecurity',
5909         'only_matching': True,
5910     }, {
5911         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5912         'only_matching': True,
5913     }, {
5914         'url': 'https://www.youtube.com/TheYoungTurks/live',
5915         'only_matching': True,
5916     }, {
5917         'url': 'https://www.youtube.com/hashtag/cctv9',
5918         'info_dict': {
5919             'id': 'cctv9',
5920             'title': 'cctv9 - All',
5921             'tags': [],
5922         },
5923         'playlist_mincount': 300,  # not consistent but should be over 300
5924     }, {
5925         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5926         'only_matching': True,
5927     }, {
5928         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5929         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5930         'only_matching': True
5931     }, {
5932         'note': '/browse/ should redirect to /channel/',
5933         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5934         'only_matching': True
5935     }, {
5936         'note': 'VLPL, should redirect to playlist?list=PL...',
5937         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5938         'info_dict': {
5939             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5940             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5941             'title': 'NCS : All Releases 💿',
5942             'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
5943             'modified_date': r're:\d{8}',
5944             'view_count': int,
5945             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5946             'tags': [],
5947             'channel': 'NoCopyrightSounds',
5948             'availability': 'public',
5949             'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
5950             'uploader': 'NoCopyrightSounds',
5951             'uploader_id': '@NoCopyrightSounds',
5952         },
5953         'playlist_mincount': 166,
5954         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden', 'YouTube Music is not directly supported'],
5955     }, {
5956         # TODO: fix 'unviewable' issue with this playlist when reloading with unavailable videos
5957         'note': 'Topic, should redirect to playlist?list=UU...',
5958         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5959         'info_dict': {
5960             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5961             'title': 'Uploads from Royalty Free Music - Topic',
5962             'tags': [],
5963             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5964             'channel': 'Royalty Free Music - Topic',
5965             'view_count': int,
5966             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5967             'modified_date': r're:\d{8}',
5968             'description': '',
5969             'availability': 'public',
5970             'uploader': 'Royalty Free Music - Topic',
5971         },
5972         'playlist_mincount': 101,
5973         'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
5974     }, {
5975         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
5976         # Treat as a general feed
5977         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5978         'info_dict': {
5979             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5980             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5981             'tags': [],
5982         },
5983         'playlist_mincount': 9,
5984     }, {
5985         'note': 'Youtube music Album',
5986         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5987         'info_dict': {
5988             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5989             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5990             'tags': [],
5991             'view_count': int,
5992             'description': '',
5993             'availability': 'unlisted',
5994             'modified_date': r're:\d{8}',
5995         },
5996         'playlist_count': 50,
5997         'expected_warnings': ['YouTube Music is not directly supported'],
5998     }, {
5999         'note': 'unlisted single video playlist',
6000         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
6001         'info_dict': {
6002             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
6003             'title': 'yt-dlp unlisted playlist test',
6004             'availability': 'unlisted',
6005             'tags': [],
6006             'modified_date': '20220418',
6007             'channel': 'colethedj',
6008             'view_count': int,
6009             'description': '',
6010             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
6011             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
6012             'uploader_url': 'https://www.youtube.com/@colethedj1894',
6013             'uploader_id': '@colethedj1894',
6014             'uploader': 'colethedj',
6015         },
6016         'playlist': [{
6017             'info_dict': {
6018                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
6019                 'id': 'BaW_jenozKc',
6020                 '_type': 'url',
6021                 'ie_key': 'Youtube',
6022                 'duration': 10,
6023                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
6024                 'channel_url': 'https://www.youtube.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
6025                 'view_count': int,
6026                 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc',
6027                 'channel': 'Philipp Hagemeister',
6028                 'uploader_id': '@PhilippHagemeister',
6029                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
6030                 'uploader': 'Philipp Hagemeister',
6031             }
6032         }],
6033         'playlist_count': 1,
6034         'params': {'extract_flat': True},
6035     }, {
6036         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
6037         'url': 'https://www.youtube.com/feed/recommended',
6038         'info_dict': {
6039             'id': 'recommended',
6040             'title': 'recommended',
6041             'tags': [],
6042         },
6043         'playlist_mincount': 50,
6044         'params': {
6045             'skip_download': True,
6046             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
6047         },
6048     }, {
6049         'note': 'API Fallback: /videos tab, sorted by oldest first',
6050         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
6051         'info_dict': {
6052             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
6053             'title': 'Cody\'sLab - Videos',
6054             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
6055             'channel': 'Cody\'sLab',
6056             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
6057             'tags': [],
6058             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
6059             'channel_follower_count': int
6060         },
6061         'playlist_mincount': 650,
6062         'params': {
6063             'skip_download': True,
6064             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
6065         },
6066         'skip': 'Query for sorting no longer works',
6067     }, {
6068         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
6069         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
6070         'info_dict': {
6071             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
6072             'title': 'Uploads from Royalty Free Music - Topic',
6073             'modified_date': r're:\d{8}',
6074             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
6075             'description': '',
6076             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
6077             'tags': [],
6078             'channel': 'Royalty Free Music - Topic',
6079             'view_count': int,
6080             'availability': 'public',
6081             'uploader': 'Royalty Free Music - Topic',
6082         },
6083         'playlist_mincount': 101,
6084         'params': {
6085             'skip_download': True,
6086             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
6087         },
6088         'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
6089     }, {
6090         'note': 'non-standard redirect to regional channel',
6091         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
6092         'only_matching': True
6093     }, {
6094         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
6095         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
6096         'info_dict': {
6097             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
6098             'modified_date': '20220407',
6099             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
6100             'tags': [],
6101             'availability': 'unlisted',
6102             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
6103             'channel': 'pukkandan',
6104             'description': 'Test for collaborative playlist',
6105             'title': 'yt-dlp test - collaborative playlist',
6106             'view_count': int,
6107             'uploader_url': 'https://www.youtube.com/@pukkandan',
6108             'uploader_id': '@pukkandan',
6109             'uploader': 'pukkandan',
6110         },
6111         'playlist_mincount': 2
6112     }, {
6113         'note': 'translated tab name',
6114         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
6115         'info_dict': {
6116             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6117             'tags': [],
6118             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6119             'description': 'test description',
6120             'title': 'cole-dlp-test-acc - 再生リスト',
6121             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6122             'channel': 'cole-dlp-test-acc',
6123             'uploader_url': 'https://www.youtube.com/@coletdjnz',
6124             'uploader_id': '@coletdjnz',
6125             'uploader': 'cole-dlp-test-acc',
6126         },
6127         'playlist_mincount': 1,
6128         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
6129         'expected_warnings': ['Preferring "ja"'],
6130     }, {
6131         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
6132         'note': 'preferred lang set with playlist with translated video titles',
6133         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
6134         'info_dict': {
6135             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
6136             'tags': [],
6137             'view_count': int,
6138             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6139             'channel': 'cole-dlp-test-acc',
6140             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6141             'description': 'test',
6142             'title': 'dlp test playlist',
6143             'availability': 'public',
6144             'uploader_url': 'https://www.youtube.com/@coletdjnz',
6145             'uploader_id': '@coletdjnz',
6146             'uploader': 'cole-dlp-test-acc',
6147         },
6148         'playlist_mincount': 1,
6149         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
6150         'expected_warnings': ['Preferring "ja"'],
6151     }, {
6152         # shorts audio pivot for 2GtVksBMYFM.
6153         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
6154         'info_dict': {
6155             'id': 'sfv_audio_pivot',
6156             'title': 'sfv_audio_pivot',
6157             'tags': [],
6158         },
6159         'playlist_mincount': 50,
6160
6161     }, {
6162         # Channel with a real live tab (not to be mistaken with streams tab)
6163         # Do not treat like it should redirect to live stream
6164         'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
6165         'info_dict': {
6166             'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
6167             'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
6168             'tags': [],
6169         },
6170         'playlist_mincount': 20,
6171     }, {
6172         # Tab name is not the same as tab id
6173         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
6174         'info_dict': {
6175             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6176             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
6177             'tags': [],
6178         },
6179         'playlist_mincount': 8,
6180     }, {
6181         # Home tab id is literally home. Not to get mistaken with featured
6182         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
6183         'info_dict': {
6184             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6185             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
6186             'tags': [],
6187         },
6188         'playlist_mincount': 8,
6189     }, {
6190         # Should get three playlists for videos, shorts and streams tabs
6191         'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
6192         'info_dict': {
6193             'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
6194             'title': 'Polka Ch. 尾丸ポルカ',
6195             'channel_follower_count': int,
6196             'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
6197             'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
6198             'description': 'md5:49809d8bf9da539bc48ed5d1f83c33f2',
6199             'channel': 'Polka Ch. 尾丸ポルカ',
6200             'tags': 'count:35',
6201             'uploader_url': 'https://www.youtube.com/@OmaruPolka',
6202             'uploader': 'Polka Ch. 尾丸ポルカ',
6203             'uploader_id': '@OmaruPolka',
6204             'channel_is_verified': True,
6205         },
6206         'playlist_count': 3,
6207     }, {
6208         # Shorts tab with channel with handle
6209         # TODO: fix channel description
6210         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
6211         'info_dict': {
6212             'id': 'UC0intLFzLaudFG-xAvUEO-A',
6213             'title': 'Not Just Bikes - Shorts',
6214             'tags': 'count:10',
6215             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
6216             'description': 'md5:5e82545b3a041345927a92d0585df247',
6217             'channel_follower_count': int,
6218             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
6219             'channel': 'Not Just Bikes',
6220             'uploader_url': 'https://www.youtube.com/@NotJustBikes',
6221             'uploader': 'Not Just Bikes',
6222             'uploader_id': '@NotJustBikes',
6223             'channel_is_verified': True,
6224         },
6225         'playlist_mincount': 10,
6226     }, {
6227         # Streams tab
6228         'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
6229         'info_dict': {
6230             'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
6231             'title': '中村悠一 - Live',
6232             'tags': 'count:7',
6233             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
6234             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
6235             'channel': '中村悠一',
6236             'channel_follower_count': int,
6237             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
6238             'uploader_url': 'https://www.youtube.com/@Yuichi-Nakamura',
6239             'uploader_id': '@Yuichi-Nakamura',
6240             'uploader': '中村悠一',
6241         },
6242         'playlist_mincount': 60,
6243     }, {
6244         # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
6245         # See test_youtube_lists
6246         'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
6247         'only_matching': True,
6248     }, {
6249         # No uploads and no UCID given. Should fail with no uploads error
6250         # See test_youtube_lists
6251         'url': 'https://www.youtube.com/news',
6252         'only_matching': True
6253     }, {
6254         # No videos tab but has a shorts tab
6255         'url': 'https://www.youtube.com/c/TKFShorts',
6256         'info_dict': {
6257             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6258             'title': 'Shorts Break - Shorts',
6259             'tags': 'count:48',
6260             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6261             'channel': 'Shorts Break',
6262             'description': 'md5:6de33c5e7ba686e5f3efd4e19c7ef499',
6263             'channel_follower_count': int,
6264             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
6265             'uploader_url': 'https://www.youtube.com/@ShortsBreak_Official',
6266             'uploader': 'Shorts Break',
6267             'uploader_id': '@ShortsBreak_Official',
6268         },
6269         'playlist_mincount': 30,
6270     }, {
6271         # Trending Now Tab. tab id is empty
6272         'url': 'https://www.youtube.com/feed/trending',
6273         'info_dict': {
6274             'id': 'trending',
6275             'title': 'trending - Now',
6276             'tags': [],
6277         },
6278         'playlist_mincount': 30,
6279     }, {
6280         # Trending Gaming Tab. tab id is empty
6281         'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
6282         'info_dict': {
6283             'id': 'trending',
6284             'title': 'trending - Gaming',
6285             'tags': [],
6286         },
6287         'playlist_mincount': 30,
6288     }, {
6289         # Shorts url result in shorts tab
6290         # TODO: Fix channel id extraction
6291         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
6292         'info_dict': {
6293             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6294             'title': 'cole-dlp-test-acc - Shorts',
6295             'channel': 'cole-dlp-test-acc',
6296             'description': 'test description',
6297             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6298             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6299             'tags': [],
6300             'uploader_url': 'https://www.youtube.com/@coletdjnz',
6301             'uploader_id': '@coletdjnz',
6302             'uploader': 'cole-dlp-test-acc',
6303         },
6304         'playlist': [{
6305             'info_dict': {
6306                 # Channel data is not currently available for short renderers (as of 2023-03-01)
6307                 '_type': 'url',
6308                 'ie_key': 'Youtube',
6309                 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
6310                 'id': 'sSM9J5YH_60',
6311                 'title': 'SHORT short',
6312                 'view_count': int,
6313                 'thumbnails': list,
6314             }
6315         }],
6316         'params': {'extract_flat': True},
6317     }, {
6318         # Live video status should be extracted
6319         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
6320         'info_dict': {
6321             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6322             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live',  # TODO, should be Minecraft - Live or Minecraft - Topic - Live
6323             'tags': []
6324         },
6325         'playlist': [{
6326             'info_dict': {
6327                 '_type': 'url',
6328                 'ie_key': 'Youtube',
6329                 'url': 'startswith:https://www.youtube.com/watch?v=',
6330                 'id': str,
6331                 'title': str,
6332                 'live_status': 'is_live',
6333                 'channel_id': str,
6334                 'channel_url': str,
6335                 'concurrent_view_count': int,
6336                 'channel': str,
6337                 'uploader': str,
6338                 'uploader_url': str,
6339                 'uploader_id': str,
6340                 'channel_is_verified': bool,  # this will keep changing
6341             }
6342         }],
6343         'params': {'extract_flat': True, 'playlist_items': '1'},
6344         'playlist_mincount': 1
6345     }, {
6346         # Channel renderer metadata. Contains number of videos on the channel
6347         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
6348         'info_dict': {
6349             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6350             'title': 'cole-dlp-test-acc - Channels',
6351             'channel': 'cole-dlp-test-acc',
6352             'description': 'test description',
6353             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6354             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6355             'tags': [],
6356             'uploader_url': 'https://www.youtube.com/@coletdjnz',
6357             'uploader_id': '@coletdjnz',
6358             'uploader': 'cole-dlp-test-acc',
6359         },
6360         'playlist': [{
6361             'info_dict': {
6362                 '_type': 'url',
6363                 'ie_key': 'YoutubeTab',
6364                 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6365                 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6366                 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6367                 'title': 'PewDiePie',
6368                 'channel': 'PewDiePie',
6369                 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6370                 'thumbnails': list,
6371                 'channel_follower_count': int,
6372                 'playlist_count': int,
6373                 'uploader': 'PewDiePie',
6374                 'uploader_url': 'https://www.youtube.com/@PewDiePie',
6375                 'uploader_id': '@PewDiePie',
6376                 'channel_is_verified': True,
6377             }
6378         }],
6379         'params': {'extract_flat': True},
6380     }, {
6381         'url': 'https://www.youtube.com/@3blue1brown/about',
6382         'info_dict': {
6383             'id': '@3blue1brown',
6384             'tags': ['Mathematics'],
6385             'title': '3Blue1Brown',
6386             'channel_follower_count': int,
6387             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
6388             'channel': '3Blue1Brown',
6389             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
6390             'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
6391             'uploader_url': 'https://www.youtube.com/@3blue1brown',
6392             'uploader_id': '@3blue1brown',
6393             'uploader': '3Blue1Brown',
6394             'channel_is_verified': True,
6395         },
6396         'playlist_count': 0,
6397     }, {
6398         # Podcasts tab, with rich entry playlistRenderers
6399         'url': 'https://www.youtube.com/@99percentinvisiblepodcast/podcasts',
6400         'info_dict': {
6401             'id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
6402             'channel_id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
6403             'uploader_url': 'https://www.youtube.com/@99percentinvisiblepodcast',
6404             'description': 'md5:3a0ed38f1ad42a68ef0428c04a15695c',
6405             'title': '99 Percent Invisible - Podcasts',
6406             'uploader': '99 Percent Invisible',
6407             'channel_follower_count': int,
6408             'channel_url': 'https://www.youtube.com/channel/UCVMF2HD4ZgC0QHpU9Yq5Xrw',
6409             'tags': [],
6410             'channel': '99 Percent Invisible',
6411             'uploader_id': '@99percentinvisiblepodcast',
6412         },
6413         'playlist_count': 0,
6414     }, {
6415         # Releases tab, with rich entry playlistRenderers (same as Podcasts tab)
6416         'url': 'https://www.youtube.com/@AHimitsu/releases',
6417         'info_dict': {
6418             'id': 'UCgFwu-j5-xNJml2FtTrrB3A',
6419             'channel': 'A Himitsu',
6420             'uploader_url': 'https://www.youtube.com/@AHimitsu',
6421             'title': 'A Himitsu - Releases',
6422             'uploader_id': '@AHimitsu',
6423             'uploader': 'A Himitsu',
6424             'channel_id': 'UCgFwu-j5-xNJml2FtTrrB3A',
6425             'tags': 'count:12',
6426             'description': 'I make music',
6427             'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
6428             'channel_follower_count': int,
6429             'channel_is_verified': True,
6430         },
6431         'playlist_mincount': 10,
6432     }, {
6433         # Playlist with only shorts, shown as reel renderers
6434         # FIXME: future: YouTube currently doesn't give continuation for this,
6435         # may do in future.
6436         'url': 'https://www.youtube.com/playlist?list=UUxqPAgubo4coVn9Lx1FuKcg',
6437         'info_dict': {
6438             'id': 'UUxqPAgubo4coVn9Lx1FuKcg',
6439             'channel_url': 'https://www.youtube.com/channel/UCxqPAgubo4coVn9Lx1FuKcg',
6440             'view_count': int,
6441             'uploader_id': '@BangyShorts',
6442             'description': '',
6443             'uploader_url': 'https://www.youtube.com/@BangyShorts',
6444             'channel_id': 'UCxqPAgubo4coVn9Lx1FuKcg',
6445             'channel': 'Bangy Shorts',
6446             'uploader': 'Bangy Shorts',
6447             'tags': [],
6448             'availability': 'public',
6449             'modified_date': r're:\d{8}',
6450             'title': 'Uploads from Bangy Shorts',
6451         },
6452         'playlist_mincount': 100,
6453         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6454     }, {
6455         'note': 'Tags containing spaces',
6456         'url': 'https://www.youtube.com/channel/UC7_YxT-KID8kRbqZo7MyscQ',
6457         'playlist_count': 3,
6458         'info_dict': {
6459             'id': 'UC7_YxT-KID8kRbqZo7MyscQ',
6460             'channel': 'Markiplier',
6461             'channel_id': 'UC7_YxT-KID8kRbqZo7MyscQ',
6462             'title': 'Markiplier',
6463             'channel_follower_count': int,
6464             'description': 'md5:0c010910558658824402809750dc5d97',
6465             'uploader_id': '@markiplier',
6466             'uploader_url': 'https://www.youtube.com/@markiplier',
6467             'uploader': 'Markiplier',
6468             'channel_url': 'https://www.youtube.com/channel/UC7_YxT-KID8kRbqZo7MyscQ',
6469             'channel_is_verified': True,
6470             'tags': ['markiplier', 'comedy', 'gaming', 'funny videos', 'funny moments',
6471                      'sketch comedy', 'laughing', 'lets play', 'challenge videos', 'hilarious',
6472                      'challenges', 'sketches', 'scary games', 'funny games', 'rage games',
6473                      'mark fischbach'],
6474         },
6475     }]
6476
6477     @classmethod
6478     def suitable(cls, url):
6479         return False if YoutubeIE.suitable(url) else super().suitable(url)
6480
6481     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
6482
6483     def _get_url_mobj(self, url):
6484         mobj = self._URL_RE.match(url).groupdict()
6485         mobj.update((k, '') for k, v in mobj.items() if v is None)
6486         return mobj
6487
6488     def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
6489         tab_name = (tab.get('title') or '').lower()
6490         tab_url = urljoin(base_url, traverse_obj(
6491             tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
6492
6493         tab_id = (tab_url and self._get_url_mobj(tab_url)['tab'][1:]
6494                   or traverse_obj(tab, 'tabIdentifier', expected_type=str))
6495         if tab_id:
6496             return {
6497                 'TAB_ID_SPONSORSHIPS': 'membership',
6498             }.get(tab_id, tab_id), tab_name
6499
6500         # Fallback to tab name if we cannot get the tab id.
6501         # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
6502         # Note that in the case of translated tab name this may result in an empty string, which we don't want.
6503         if tab_name:
6504             self.write_debug(f'Falling back to selected tab name: {tab_name}')
6505         return {
6506             'home': 'featured',
6507             'live': 'streams',
6508         }.get(tab_name, tab_name), tab_name
6509
6510     def _has_tab(self, tabs, tab_id):
6511         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
6512
6513     def _empty_playlist(self, item_id, data):
6514         return self.playlist_result([], item_id, **self._extract_metadata_from_tabs(item_id, data))
6515
6516     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
6517     def _real_extract(self, url, smuggled_data):
6518         item_id = self._match_id(url)
6519         url = urllib.parse.urlunparse(
6520             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
6521         compat_opts = self.get_param('compat_opts', [])
6522
6523         mobj = self._get_url_mobj(url)
6524         pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
6525         if is_channel and smuggled_data.get('is_music_url'):
6526             if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
6527                 return self.url_result(
6528                     f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
6529             elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
6530                 mdata = self._extract_tab_endpoint(
6531                     f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
6532                 murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
6533                                     get_all=False, expected_type=str)
6534                 if not murl:
6535                     raise ExtractorError('Failed to resolve album to playlist')
6536                 return self.url_result(murl, YoutubeTabIE)
6537             elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
6538                 return self.url_result(
6539                     f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
6540
6541         original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
6542         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
6543             url = f'{pre}/videos{post}'
6544         if smuggled_data.get('is_music_url'):
6545             self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
6546
6547         # Handle both video/playlist URLs
6548         qs = parse_qs(url)
6549         video_id, playlist_id = [traverse_obj(qs, (key, 0)) for key in ('v', 'list')]
6550         if not video_id and mobj['not_channel'].startswith('watch'):
6551             if not playlist_id:
6552                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
6553                 raise ExtractorError('A video URL was given without video ID', expected=True)
6554             # Common mistake: https://www.youtube.com/watch?list=playlist_id
6555             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
6556             return self.url_result(
6557                 f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
6558
6559         if not self._yes_playlist(playlist_id, video_id):
6560             return self.url_result(
6561                 f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6562
6563         data, ytcfg = self._extract_data(url, display_id)
6564
6565         # YouTube may provide a non-standard redirect to the regional channel
6566         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
6567         # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
6568         redirect_url = traverse_obj(
6569             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
6570         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
6571             redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
6572             self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
6573             return self.url_result(redirect_url, YoutubeTabIE)
6574
6575         tabs, extra_tabs = self._extract_tab_renderers(data), []
6576         if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
6577             selected_tab = self._extract_selected_tab(tabs)
6578             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
6579             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
6580
6581             # /about is no longer a tab
6582             if original_tab_id == 'about':
6583                 return self._empty_playlist(item_id, data)
6584
6585             if not original_tab_id and selected_tab_name:
6586                 self.to_screen('Downloading all uploads of the channel. '
6587                                'To download only the videos in a specific tab, pass the tab\'s URL')
6588                 if self._has_tab(tabs, 'streams'):
6589                     extra_tabs.append(''.join((pre, '/streams', post)))
6590                 if self._has_tab(tabs, 'shorts'):
6591                     extra_tabs.append(''.join((pre, '/shorts', post)))
6592                 # XXX: Members-only tab should also be extracted
6593
6594                 if not extra_tabs and selected_tab_id != 'videos':
6595                     # Channel does not have streams, shorts or videos tabs
6596                     if item_id[:2] != 'UC':
6597                         return self._empty_playlist(item_id, data)
6598
6599                     # Topic channels don't have /videos. Use the equivalent playlist instead
6600                     pl_id = f'UU{item_id[2:]}'
6601                     pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
6602                     try:
6603                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
6604                     except ExtractorError:
6605                         return self._empty_playlist(item_id, data)
6606                     else:
6607                         item_id, url = pl_id, pl_url
6608                         self.to_screen(
6609                             f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
6610
6611                 elif extra_tabs and selected_tab_id != 'videos':
6612                     # When there are shorts/live tabs but not videos tab
6613                     url, data = f'{pre}{post}', None
6614
6615             elif (original_tab_id or 'videos') != selected_tab_id:
6616                 if original_tab_id == 'live':
6617                     # Live tab should have redirected to the video
6618                     # Except in the case the channel has an actual live tab
6619                     # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
6620                     raise UserNotLive(video_id=item_id)
6621                 elif selected_tab_name:
6622                     raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
6623
6624                 # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
6625                 url = f'{pre}{post}'
6626
6627         # YouTube sometimes provides a button to reload playlist with unavailable videos.
6628         if 'no-youtube-unavailable-videos' not in compat_opts:
6629             data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
6630         self._extract_and_report_alerts(data, only_once=True)
6631
6632         tabs, entries = self._extract_tab_renderers(data), []
6633         if tabs:
6634             entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
6635             entries[0].update({
6636                 'extractor_key': YoutubeTabIE.ie_key(),
6637                 'extractor': YoutubeTabIE.IE_NAME,
6638                 'webpage_url': url,
6639             })
6640         if self.get_param('playlist_items') == '0':
6641             entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
6642         else:  # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
6643             entries.extend(map(self._real_extract, extra_tabs))
6644
6645         if len(entries) == 1:
6646             return entries[0]
6647         elif entries:
6648             metadata = self._extract_metadata_from_tabs(item_id, data)
6649             uploads_url = 'the Uploads (UU) playlist URL'
6650             if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
6651                 uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
6652             self.to_screen(
6653                 'Downloading as multiple playlists, separated by tabs. '
6654                 f'To download as a single playlist instead, pass {uploads_url}')
6655             return self.playlist_result(entries, item_id, **metadata)
6656
6657         # Inline playlist
6658         playlist = traverse_obj(
6659             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
6660         if playlist:
6661             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
6662
6663         video_id = traverse_obj(
6664             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
6665         if video_id:
6666             if tab != '/live':  # live tab is expected to redirect to video
6667                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
6668             return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6669
6670         raise ExtractorError('Unable to recognize tab page')
6671
6672
6673 class YoutubePlaylistIE(InfoExtractor):
6674     IE_DESC = 'YouTube playlists'
6675     _VALID_URL = r'''(?x)(?:
6676                         (?:https?://)?
6677                         (?:\w+\.)?
6678                         (?:
6679                             (?:
6680                                 youtube(?:kids)?\.com|
6681                                 %(invidious)s
6682                             )
6683                             /.*?\?.*?\blist=
6684                         )?
6685                         (?P<id>%(playlist_id)s)
6686                      )''' % {
6687         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
6688         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
6689     }
6690     IE_NAME = 'youtube:playlist'
6691     _TESTS = [{
6692         'note': 'issue #673',
6693         'url': 'PLBB231211A4F62143',
6694         'info_dict': {
6695             'title': '[OLD]Team Fortress 2 (Class-based LP)',
6696             'id': 'PLBB231211A4F62143',
6697             'uploader': 'Wickman',
6698             'uploader_id': '@WickmanVT',
6699             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
6700             'view_count': int,
6701             'uploader_url': 'https://www.youtube.com/@WickmanVT',
6702             'modified_date': r're:\d{8}',
6703             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6704             'channel': 'Wickman',
6705             'tags': [],
6706             'channel_url': 'https://www.youtube.com/channel/UCKSpbfbl5kRQpTdL7kMc-1Q',
6707             'availability': 'public',
6708         },
6709         'playlist_mincount': 29,
6710     }, {
6711         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6712         'info_dict': {
6713             'title': 'YDL_safe_search',
6714             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6715         },
6716         'playlist_count': 2,
6717         'skip': 'This playlist is private',
6718     }, {
6719         'note': 'embedded',
6720         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6721         'playlist_count': 4,
6722         'info_dict': {
6723             'title': 'JODA15',
6724             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6725             'uploader': 'milan',
6726             'uploader_id': '@milan5503',
6727             'description': '',
6728             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6729             'tags': [],
6730             'modified_date': '20140919',
6731             'view_count': int,
6732             'channel': 'milan',
6733             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6734             'uploader_url': 'https://www.youtube.com/@milan5503',
6735             'availability': 'public',
6736         },
6737         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden', 'Retrying', 'Giving up'],
6738     }, {
6739         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6740         'playlist_mincount': 455,
6741         'info_dict': {
6742             'title': '2018 Chinese New Singles (11/6 updated)',
6743             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6744             'uploader': 'LBK',
6745             'uploader_id': '@music_king',
6746             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
6747             'channel': 'LBK',
6748             'view_count': int,
6749             'channel_url': 'https://www.youtube.com/channel/UC21nz3_MesPLqtDqwdvnoxA',
6750             'tags': [],
6751             'uploader_url': 'https://www.youtube.com/@music_king',
6752             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6753             'modified_date': r're:\d{8}',
6754             'availability': 'public',
6755         },
6756         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6757     }, {
6758         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
6759         'only_matching': True,
6760     }, {
6761         # music album playlist
6762         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
6763         'only_matching': True,
6764     }]
6765
6766     @classmethod
6767     def suitable(cls, url):
6768         if YoutubeTabIE.suitable(url):
6769             return False
6770         from ..utils import parse_qs
6771         qs = parse_qs(url)
6772         if qs.get('v', [None])[0]:
6773             return False
6774         return super().suitable(url)
6775
6776     def _real_extract(self, url):
6777         playlist_id = self._match_id(url)
6778         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
6779         url = update_url_query(
6780             'https://www.youtube.com/playlist',
6781             parse_qs(url) or {'list': playlist_id})
6782         if is_music_url:
6783             url = smuggle_url(url, {'is_music_url': True})
6784         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6785
6786
6787 class YoutubeYtBeIE(InfoExtractor):
6788     IE_DESC = 'youtu.be'
6789     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
6790     _TESTS = [{
6791         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
6792         'info_dict': {
6793             'id': 'yeWKywCrFtk',
6794             'ext': 'mp4',
6795             'title': 'Small Scale Baler and Braiding Rugs',
6796             'uploader': 'Backus-Page House Museum',
6797             'uploader_id': '@backuspagemuseum',
6798             'uploader_url': r're:https?://(?:www\.)?youtube\.com/@backuspagemuseum',
6799             'upload_date': '20161008',
6800             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
6801             'categories': ['Nonprofits & Activism'],
6802             'tags': list,
6803             'like_count': int,
6804             'age_limit': 0,
6805             'playable_in_embed': True,
6806             'thumbnail': r're:^https?://.*\.webp',
6807             'channel': 'Backus-Page House Museum',
6808             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
6809             'live_status': 'not_live',
6810             'view_count': int,
6811             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
6812             'availability': 'public',
6813             'duration': 59,
6814             'comment_count': int,
6815             'channel_follower_count': int
6816         },
6817         'params': {
6818             'noplaylist': True,
6819             'skip_download': True,
6820         },
6821     }, {
6822         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6823         'only_matching': True,
6824     }]
6825
6826     def _real_extract(self, url):
6827         mobj = self._match_valid_url(url)
6828         video_id = mobj.group('id')
6829         playlist_id = mobj.group('playlist_id')
6830         return self.url_result(
6831             update_url_query('https://www.youtube.com/watch', {
6832                 'v': video_id,
6833                 'list': playlist_id,
6834                 'feature': 'youtu.be',
6835             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6836
6837
6838 class YoutubeLivestreamEmbedIE(InfoExtractor):
6839     IE_DESC = 'YouTube livestream embeds'
6840     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6841     _TESTS = [{
6842         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6843         'only_matching': True,
6844     }]
6845
6846     def _real_extract(self, url):
6847         channel_id = self._match_id(url)
6848         return self.url_result(
6849             f'https://www.youtube.com/channel/{channel_id}/live',
6850             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6851
6852
6853 class YoutubeYtUserIE(InfoExtractor):
6854     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6855     IE_NAME = 'youtube:user'
6856     _VALID_URL = r'ytuser:(?P<id>.+)'
6857     _TESTS = [{
6858         'url': 'ytuser:phihag',
6859         'only_matching': True,
6860     }]
6861
6862     def _real_extract(self, url):
6863         user_id = self._match_id(url)
6864         return self.url_result(f'https://www.youtube.com/user/{user_id}', YoutubeTabIE, user_id)
6865
6866
6867 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6868     IE_NAME = 'youtube:favorites'
6869     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6870     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6871     _LOGIN_REQUIRED = True
6872     _TESTS = [{
6873         'url': ':ytfav',
6874         'only_matching': True,
6875     }, {
6876         'url': ':ytfavorites',
6877         'only_matching': True,
6878     }]
6879
6880     def _real_extract(self, url):
6881         return self.url_result(
6882             'https://www.youtube.com/playlist?list=LL',
6883             ie=YoutubeTabIE.ie_key())
6884
6885
6886 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6887     IE_NAME = 'youtube:notif'
6888     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6889     _VALID_URL = r':ytnotif(?:ication)?s?'
6890     _LOGIN_REQUIRED = True
6891     _TESTS = [{
6892         'url': ':ytnotif',
6893         'only_matching': True,
6894     }, {
6895         'url': ':ytnotifications',
6896         'only_matching': True,
6897     }]
6898
6899     def _extract_notification_menu(self, response, continuation_list):
6900         notification_list = traverse_obj(
6901             response,
6902             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6903             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6904             expected_type=list) or []
6905         continuation_list[0] = None
6906         for item in notification_list:
6907             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6908             if entry:
6909                 yield entry
6910             continuation = item.get('continuationItemRenderer')
6911             if continuation:
6912                 continuation_list[0] = continuation
6913
6914     def _extract_notification_renderer(self, notification):
6915         video_id = traverse_obj(
6916             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6917         url = f'https://www.youtube.com/watch?v={video_id}'
6918         channel_id = None
6919         if not video_id:
6920             browse_ep = traverse_obj(
6921                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6922             channel_id = self.ucid_or_none(traverse_obj(browse_ep, 'browseId', expected_type=str))
6923             post_id = self._search_regex(
6924                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6925                 'post id', default=None)
6926             if not channel_id or not post_id:
6927                 return
6928             # The direct /post url redirects to this in the browser
6929             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6930
6931         channel = traverse_obj(
6932             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6933             expected_type=str)
6934         notification_title = self._get_text(notification, 'shortMessage')
6935         if notification_title:
6936             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6937         # TODO: handle recommended videos
6938         title = self._search_regex(
6939             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6940             'video title', default=None)
6941         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6942                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6943                      else None)
6944         return {
6945             '_type': 'url',
6946             'url': url,
6947             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6948             'video_id': video_id,
6949             'title': title,
6950             'channel_id': channel_id,
6951             'channel': channel,
6952             'uploader': channel,
6953             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6954             'timestamp': timestamp,
6955         }
6956
6957     def _notification_menu_entries(self, ytcfg):
6958         continuation_list = [None]
6959         response = None
6960         for page in itertools.count(1):
6961             ctoken = traverse_obj(
6962                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6963             response = self._extract_response(
6964                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6965                 ep='notification/get_notification_menu', check_get_keys='actions',
6966                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6967             yield from self._extract_notification_menu(response, continuation_list)
6968             if not continuation_list[0]:
6969                 break
6970
6971     def _real_extract(self, url):
6972         display_id = 'notifications'
6973         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6974         self._report_playlist_authcheck(ytcfg)
6975         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6976
6977
6978 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6979     IE_DESC = 'YouTube search'
6980     IE_NAME = 'youtube:search'
6981     _SEARCH_KEY = 'ytsearch'
6982     _SEARCH_PARAMS = 'EgIQAfABAQ=='  # Videos only
6983     _TESTS = [{
6984         'url': 'ytsearch5:youtube-dl test video',
6985         'playlist_count': 5,
6986         'info_dict': {
6987             'id': 'youtube-dl test video',
6988             'title': 'youtube-dl test video',
6989         }
6990     }, {
6991         'note': 'Suicide/self-harm search warning',
6992         'url': 'ytsearch1:i hate myself and i wanna die',
6993         'playlist_count': 1,
6994         'info_dict': {
6995             'id': 'i hate myself and i wanna die',
6996             'title': 'i hate myself and i wanna die',
6997         }
6998     }]
6999
7000
7001 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
7002     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
7003     _SEARCH_KEY = 'ytsearchdate'
7004     IE_DESC = 'YouTube search, newest videos first'
7005     _SEARCH_PARAMS = 'CAISAhAB8AEB'  # Videos only, sorted by date
7006     _TESTS = [{
7007         'url': 'ytsearchdate5:youtube-dl test video',
7008         'playlist_count': 5,
7009         'info_dict': {
7010             'id': 'youtube-dl test video',
7011             'title': 'youtube-dl test video',
7012         }
7013     }]
7014
7015
7016 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
7017     IE_DESC = 'YouTube search URLs with sorting and filter support'
7018     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
7019     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
7020     _TESTS = [{
7021         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
7022         'playlist_mincount': 5,
7023         'info_dict': {
7024             'id': 'youtube-dl test video',
7025             'title': 'youtube-dl test video',
7026         }
7027     }, {
7028         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
7029         'playlist_mincount': 5,
7030         'info_dict': {
7031             'id': 'python',
7032             'title': 'python',
7033         }
7034     }, {
7035         'url': 'https://www.youtube.com/results?search_query=%23cats',
7036         'playlist_mincount': 1,
7037         'info_dict': {
7038             'id': '#cats',
7039             'title': '#cats',
7040             # The test suite does not have support for nested playlists
7041             # 'entries': [{
7042             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
7043             #     'title': '#cats',
7044             # }],
7045         },
7046     }, {
7047         # Channel results
7048         'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
7049         'info_dict': {
7050             'id': 'kurzgesagt',
7051             'title': 'kurzgesagt',
7052         },
7053         'playlist': [{
7054             'info_dict': {
7055                 '_type': 'url',
7056                 'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
7057                 'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
7058                 'ie_key': 'YoutubeTab',
7059                 'channel': 'Kurzgesagt – In a Nutshell',
7060                 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
7061                 'title': 'Kurzgesagt – In a Nutshell',
7062                 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
7063                 # No longer available for search as it is set to the handle.
7064                 # 'playlist_count': int,
7065                 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
7066                 'thumbnails': list,
7067                 'uploader_id': '@kurzgesagt',
7068                 'uploader_url': 'https://www.youtube.com/@kurzgesagt',
7069                 'uploader': 'Kurzgesagt – In a Nutshell',
7070                 'channel_is_verified': True,
7071                 'channel_follower_count': int,
7072             }
7073         }],
7074         'params': {'extract_flat': True, 'playlist_items': '1'},
7075         'playlist_mincount': 1,
7076     }, {
7077         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
7078         'only_matching': True,
7079     }]
7080
7081     def _real_extract(self, url):
7082         qs = parse_qs(url)
7083         query = (qs.get('search_query') or qs.get('q'))[0]
7084         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
7085
7086
7087 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
7088     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
7089     IE_NAME = 'youtube:music:search_url'
7090     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
7091     _TESTS = [{
7092         'url': 'https://music.youtube.com/search?q=royalty+free+music',
7093         'playlist_count': 16,
7094         'info_dict': {
7095             'id': 'royalty free music',
7096             'title': 'royalty free music',
7097         }
7098     }, {
7099         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
7100         'playlist_mincount': 30,
7101         'info_dict': {
7102             'id': 'royalty free music - songs',
7103             'title': 'royalty free music - songs',
7104         },
7105         'params': {'extract_flat': 'in_playlist'}
7106     }, {
7107         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
7108         'playlist_mincount': 30,
7109         'info_dict': {
7110             'id': 'royalty free music - community playlists',
7111             'title': 'royalty free music - community playlists',
7112         },
7113         'params': {'extract_flat': 'in_playlist'}
7114     }]
7115
7116     _SECTIONS = {
7117         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
7118         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
7119         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
7120         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
7121         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
7122         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
7123     }
7124
7125     def _real_extract(self, url):
7126         qs = parse_qs(url)
7127         query = (qs.get('search_query') or qs.get('q'))[0]
7128         params = qs.get('sp', (None,))[0]
7129         if params:
7130             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
7131         else:
7132             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
7133             params = self._SECTIONS.get(section)
7134             if not params:
7135                 section = None
7136         title = join_nonempty(query, section, delim=' - ')
7137         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
7138
7139
7140 class YoutubeFeedsInfoExtractor(InfoExtractor):
7141     """
7142     Base class for feed extractors
7143     Subclasses must re-define the _FEED_NAME property.
7144     """
7145     _LOGIN_REQUIRED = True
7146     _FEED_NAME = 'feeds'
7147
7148     def _real_initialize(self):
7149         YoutubeBaseInfoExtractor._check_login_required(self)
7150
7151     @classproperty
7152     def IE_NAME(self):
7153         return f'youtube:{self._FEED_NAME}'
7154
7155     def _real_extract(self, url):
7156         return self.url_result(
7157             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
7158
7159
7160 class YoutubeWatchLaterIE(InfoExtractor):
7161     IE_NAME = 'youtube:watchlater'
7162     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
7163     _VALID_URL = r':ytwatchlater'
7164     _TESTS = [{
7165         'url': ':ytwatchlater',
7166         'only_matching': True,
7167     }]
7168
7169     def _real_extract(self, url):
7170         return self.url_result(
7171             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
7172
7173
7174 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
7175     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
7176     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
7177     _FEED_NAME = 'recommended'
7178     _LOGIN_REQUIRED = False
7179     _TESTS = [{
7180         'url': ':ytrec',
7181         'only_matching': True,
7182     }, {
7183         'url': ':ytrecommended',
7184         'only_matching': True,
7185     }, {
7186         'url': 'https://youtube.com',
7187         'only_matching': True,
7188     }]
7189
7190
7191 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
7192     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
7193     _VALID_URL = r':ytsub(?:scription)?s?'
7194     _FEED_NAME = 'subscriptions'
7195     _TESTS = [{
7196         'url': ':ytsubs',
7197         'only_matching': True,
7198     }, {
7199         'url': ':ytsubscriptions',
7200         'only_matching': True,
7201     }]
7202
7203
7204 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
7205     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
7206     _VALID_URL = r':ythis(?:tory)?'
7207     _FEED_NAME = 'history'
7208     _TESTS = [{
7209         'url': ':ythistory',
7210         'only_matching': True,
7211     }]
7212
7213
7214 class YoutubeShortsAudioPivotIE(InfoExtractor):
7215     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
7216     IE_NAME = 'youtube:shorts:pivot:audio'
7217     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
7218     _TESTS = [{
7219         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
7220         'only_matching': True,
7221     }]
7222
7223     @staticmethod
7224     def _generate_audio_pivot_params(video_id):
7225         """
7226         Generates sfv_audio_pivot browse params for this video id
7227         """
7228         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
7229         return urllib.parse.quote(base64.b64encode(pb_params).decode())
7230
7231     def _real_extract(self, url):
7232         video_id = self._match_id(url)
7233         return self.url_result(
7234             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
7235             ie=YoutubeTabIE)
7236
7237
7238 class YoutubeTruncatedURLIE(InfoExtractor):
7239     IE_NAME = 'youtube:truncated_url'
7240     IE_DESC = False  # Do not list
7241     _VALID_URL = r'''(?x)
7242         (?:https?://)?
7243         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
7244         (?:watch\?(?:
7245             feature=[a-z_]+|
7246             annotation_id=annotation_[^&]+|
7247             x-yt-cl=[0-9]+|
7248             hl=[^&]*|
7249             t=[0-9]+
7250         )?
7251         |
7252             attribution_link\?a=[^&]+
7253         )
7254         $
7255     '''
7256
7257     _TESTS = [{
7258         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
7259         'only_matching': True,
7260     }, {
7261         'url': 'https://www.youtube.com/watch?',
7262         'only_matching': True,
7263     }, {
7264         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
7265         'only_matching': True,
7266     }, {
7267         'url': 'https://www.youtube.com/watch?feature=foo',
7268         'only_matching': True,
7269     }, {
7270         'url': 'https://www.youtube.com/watch?hl=en-GB',
7271         'only_matching': True,
7272     }, {
7273         'url': 'https://www.youtube.com/watch?t=2372',
7274         'only_matching': True,
7275     }]
7276
7277     def _real_extract(self, url):
7278         raise ExtractorError(
7279             'Did you forget to quote the URL? Remember that & is a meta '
7280             'character in most shells, so you want to put the URL in quotes, '
7281             'like  youtube-dl '
7282             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
7283             ' or simply  youtube-dl BaW_jenozKc  .',
7284             expected=True)
7285
7286
7287 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
7288     IE_NAME = 'youtube:clip'
7289     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
7290     _TESTS = [{
7291         # FIXME: Other metadata should be extracted from the clip, not from the base video
7292         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
7293         'info_dict': {
7294             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
7295             'ext': 'mp4',
7296             'section_start': 29.0,
7297             'section_end': 39.7,
7298             'duration': 10.7,
7299             'age_limit': 0,
7300             'availability': 'public',
7301             'categories': ['Gaming'],
7302             'channel': 'Scott The Woz',
7303             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
7304             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
7305             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
7306             'like_count': int,
7307             'playable_in_embed': True,
7308             'tags': 'count:17',
7309             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
7310             'title': 'Mobile Games on Console - Scott The Woz',
7311             'upload_date': '20210920',
7312             'uploader': 'Scott The Woz',
7313             'uploader_id': '@ScottTheWoz',
7314             'uploader_url': 'https://www.youtube.com/@ScottTheWoz',
7315             'view_count': int,
7316             'live_status': 'not_live',
7317             'channel_follower_count': int,
7318             'chapters': 'count:20',
7319             'comment_count': int,
7320             'heatmap': 'count:100',
7321         }
7322     }]
7323
7324     def _real_extract(self, url):
7325         clip_id = self._match_id(url)
7326         _, data = self._extract_webpage(url, clip_id)
7327
7328         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
7329         if not video_id:
7330             raise ExtractorError('Unable to find video ID')
7331
7332         clip_data = traverse_obj(data, (
7333             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
7334             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
7335             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
7336             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
7337
7338         return {
7339             '_type': 'url_transparent',
7340             'url': f'https://www.youtube.com/watch?v={video_id}',
7341             'ie_key': YoutubeIE.ie_key(),
7342             'id': clip_id,
7343             'section_start': int(clip_data['startTimeMs']) / 1000,
7344             'section_end': int(clip_data['endTimeMs']) / 1000,
7345         }
7346
7347
7348 class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
7349     IE_NAME = 'youtube:consent'
7350     IE_DESC = False  # Do not list
7351     _VALID_URL = r'https?://consent\.youtube\.com/m\?'
7352     _TESTS = [{
7353         'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
7354         'info_dict': {
7355             'id': 'qVv6vCqciTM',
7356             'ext': 'mp4',
7357             'age_limit': 0,
7358             'uploader_id': '@sana_natori',
7359             'comment_count': int,
7360             'chapters': 'count:13',
7361             'upload_date': '20221223',
7362             'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
7363             'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
7364             'uploader_url': 'https://www.youtube.com/@sana_natori',
7365             'like_count': int,
7366             'release_date': '20221223',
7367             'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
7368             'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
7369             'view_count': int,
7370             'playable_in_embed': True,
7371             'duration': 4438,
7372             'availability': 'public',
7373             'channel_follower_count': int,
7374             'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
7375             'categories': ['Entertainment'],
7376             'live_status': 'was_live',
7377             'release_timestamp': 1671793345,
7378             'channel': 'さなちゃんねる',
7379             'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
7380             'uploader': 'さなちゃんねる',
7381             'channel_is_verified': True,
7382             'heatmap': 'count:100',
7383         },
7384         'add_ie': ['Youtube'],
7385         'params': {'skip_download': 'Youtube'},
7386     }]
7387
7388     def _real_extract(self, url):
7389         redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
7390         if not redirect_url:
7391             raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
7392         return self.url_result(redirect_url)
7393
7394
7395 class YoutubeTruncatedIDIE(InfoExtractor):
7396     IE_NAME = 'youtube:truncated_id'
7397     IE_DESC = False  # Do not list
7398     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
7399
7400     _TESTS = [{
7401         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
7402         'only_matching': True,
7403     }]
7404
7405     def _real_extract(self, url):
7406         video_id = self._match_id(url)
7407         raise ExtractorError(
7408             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
7409             expected=True)