yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import enum
   6 import hashlib
   7 import itertools
   8 import json
   9 import math
  10 import os.path
  11 import random
  12 import re
  13 import sys
  14 import threading
  15 import time
  16 import traceback
  17 import urllib.error
  18 import urllib.parse
  19
  20 from .common import InfoExtractor, SearchInfoExtractor
  21 from .openload import PhantomJSwrapper
  22 from ..compat import functools
  23 from ..jsinterp import JSInterpreter
  24 from ..utils import (
  25     NO_DEFAULT,
  26     ExtractorError,
  27     LazyList,
  28     UserNotLive,
  29     bug_reports_message,
  30     classproperty,
  31     clean_html,
  32     datetime_from_str,
  33     dict_get,
  34     filter_dict,
  35     float_or_none,
  36     format_field,
  37     get_first,
  38     int_or_none,
  39     is_html,
  40     join_nonempty,
  41     js_to_json,
  42     mimetype2ext,
  43     network_exceptions,
  44     orderedSet,
  45     parse_codecs,
  46     parse_count,
  47     parse_duration,
  48     parse_iso8601,
  49     parse_qs,
  50     qualities,
  51     remove_start,
  52     smuggle_url,
  53     str_or_none,
  54     str_to_int,
  55     strftime_or_none,
  56     traverse_obj,
  57     try_get,
  58     unescapeHTML,
  59     unified_strdate,
  60     unified_timestamp,
  61     unsmuggle_url,
  62     update_url_query,
  63     url_or_none,
  64     urljoin,
  65     variadic,
  66 )
  67
  68 # any clients starting with _ cannot be explicitly requested by the user
  69 INNERTUBE_CLIENTS = {
  70     'web': {
  71         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  72         'INNERTUBE_CONTEXT': {
  73             'client': {
  74                 'clientName': 'WEB',
  75                 'clientVersion': '2.20220801.00.00',
  76             }
  77         },
  78         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  79     },
  80     'web_embedded': {
  81         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  82         'INNERTUBE_CONTEXT': {
  83             'client': {
  84                 'clientName': 'WEB_EMBEDDED_PLAYER',
  85                 'clientVersion': '1.20220731.00.00',
  86             },
  87         },
  88         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  89     },
  90     'web_music': {
  91         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  92         'INNERTUBE_HOST': 'music.youtube.com',
  93         'INNERTUBE_CONTEXT': {
  94             'client': {
  95                 'clientName': 'WEB_REMIX',
  96                 'clientVersion': '1.20220727.01.00',
  97             }
  98         },
  99         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 100     },
 101     'web_creator': {
 102         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 103         'INNERTUBE_CONTEXT': {
 104             'client': {
 105                 'clientName': 'WEB_CREATOR',
 106                 'clientVersion': '1.20220726.00.00',
 107             }
 108         },
 109         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 110     },
 111     'android': {
 112         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 113         'INNERTUBE_CONTEXT': {
 114             'client': {
 115                 'clientName': 'ANDROID',
 116                 'clientVersion': '17.31.35',
 117                 'androidSdkVersion': 30,
 118                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 119             }
 120         },
 121         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 122         'REQUIRE_JS_PLAYER': False
 123     },
 124     'android_embedded': {
 125         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 126         'INNERTUBE_CONTEXT': {
 127             'client': {
 128                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 129                 'clientVersion': '17.31.35',
 130                 'androidSdkVersion': 30,
 131                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 132             },
 133         },
 134         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 135         'REQUIRE_JS_PLAYER': False
 136     },
 137     'android_music': {
 138         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 139         'INNERTUBE_CONTEXT': {
 140             'client': {
 141                 'clientName': 'ANDROID_MUSIC',
 142                 'clientVersion': '5.16.51',
 143                 'androidSdkVersion': 30,
 144                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 145             }
 146         },
 147         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 148         'REQUIRE_JS_PLAYER': False
 149     },
 150     'android_creator': {
 151         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 152         'INNERTUBE_CONTEXT': {
 153             'client': {
 154                 'clientName': 'ANDROID_CREATOR',
 155                 'clientVersion': '22.30.100',
 156                 'androidSdkVersion': 30,
 157                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 158             },
 159         },
 160         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 161         'REQUIRE_JS_PLAYER': False
 162     },
 163     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 164     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 165     'ios': {
 166         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 167         'INNERTUBE_CONTEXT': {
 168             'client': {
 169                 'clientName': 'IOS',
 170                 'clientVersion': '17.33.2',
 171                 'deviceModel': 'iPhone14,3',
 172                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 173             }
 174         },
 175         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 176         'REQUIRE_JS_PLAYER': False
 177     },
 178     'ios_embedded': {
 179         'INNERTUBE_CONTEXT': {
 180             'client': {
 181                 'clientName': 'IOS_MESSAGES_EXTENSION',
 182                 'clientVersion': '17.33.2',
 183                 'deviceModel': 'iPhone14,3',
 184                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 185             },
 186         },
 187         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 188         'REQUIRE_JS_PLAYER': False
 189     },
 190     'ios_music': {
 191         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 192         'INNERTUBE_CONTEXT': {
 193             'client': {
 194                 'clientName': 'IOS_MUSIC',
 195                 'clientVersion': '5.21',
 196                 'deviceModel': 'iPhone14,3',
 197                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 198             },
 199         },
 200         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 201         'REQUIRE_JS_PLAYER': False
 202     },
 203     'ios_creator': {
 204         'INNERTUBE_CONTEXT': {
 205             'client': {
 206                 'clientName': 'IOS_CREATOR',
 207                 'clientVersion': '22.33.101',
 208                 'deviceModel': 'iPhone14,3',
 209                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 210             },
 211         },
 212         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 213         'REQUIRE_JS_PLAYER': False
 214     },
 215     # mweb has 'ultralow' formats
 216     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 217     'mweb': {
 218         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 219         'INNERTUBE_CONTEXT': {
 220             'client': {
 221                 'clientName': 'MWEB',
 222                 'clientVersion': '2.20220801.00.00',
 223             }
 224         },
 225         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 226     },
 227     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 228     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 229     'tv_embedded': {
 230         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 231         'INNERTUBE_CONTEXT': {
 232             'client': {
 233                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 234                 'clientVersion': '2.0',
 235             },
 236         },
 237         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 238     },
 239 }
 240
 241
 242 def _split_innertube_client(client_name):
 243     variant, *base = client_name.rsplit('.', 1)
 244     if base:
 245         return variant, base[0], variant
 246     base, *variant = client_name.split('_', 1)
 247     return client_name, base, variant[0] if variant else None
 248
 249
 250 def build_innertube_clients():
 251     THIRD_PARTY = {
 252         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 253     }
 254     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 255     priority = qualities(BASE_CLIENTS[::-1])
 256
 257     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 258         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 259         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 260         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 261         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 262
 263         _, base_client, variant = _split_innertube_client(client)
 264         ytcfg['priority'] = 10 * priority(base_client)
 265
 266         if not variant:
 267             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 268             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 269             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 270             embedscreen['priority'] -= 3
 271         elif variant == 'embedded':
 272             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 273             ytcfg['priority'] -= 2
 274         else:
 275             ytcfg['priority'] -= 3
 276
 277
 278 build_innertube_clients()
 279
 280
 281 class BadgeType(enum.Enum):
 282     AVAILABILITY_UNLISTED = enum.auto()
 283     AVAILABILITY_PRIVATE = enum.auto()
 284     AVAILABILITY_PUBLIC = enum.auto()
 285     AVAILABILITY_PREMIUM = enum.auto()
 286     AVAILABILITY_SUBSCRIPTION = enum.auto()
 287     LIVE_NOW = enum.auto()
 288
 289
 290 class YoutubeBaseInfoExtractor(InfoExtractor):
 291     """Provide base functions for Youtube extractors"""
 292
 293     _RESERVED_NAMES = (
 294         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 295         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 296         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 297         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 298
 299     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 300
 301     # _NETRC_MACHINE = 'youtube'
 302
 303     # If True it will raise an error if no login info is provided
 304     _LOGIN_REQUIRED = False
 305
 306     _INVIDIOUS_SITES = (
 307         # invidious-redirect websites
 308         r'(?:www\.)?redirect\.invidious\.io',
 309         r'(?:(?:www|dev)\.)?invidio\.us',
 310         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 311         r'(?:www\.)?invidious\.pussthecat\.org',
 312         r'(?:www\.)?invidious\.zee\.li',
 313         r'(?:www\.)?invidious\.ethibox\.fr',
 314         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 315         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 316         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 317         # youtube-dl invidious instances list
 318         r'(?:(?:www|no)\.)?invidiou\.sh',
 319         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 320         r'(?:www\.)?invidious\.kabi\.tk',
 321         r'(?:www\.)?invidious\.mastodon\.host',
 322         r'(?:www\.)?invidious\.zapashcanon\.fr',
 323         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 324         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 325         r'(?:www\.)?invidious\.himiko\.cloud',
 326         r'(?:www\.)?invidious\.reallyancient\.tech',
 327         r'(?:www\.)?invidious\.tube',
 328         r'(?:www\.)?invidiou\.site',
 329         r'(?:www\.)?invidious\.site',
 330         r'(?:www\.)?invidious\.xyz',
 331         r'(?:www\.)?invidious\.nixnet\.xyz',
 332         r'(?:www\.)?invidious\.048596\.xyz',
 333         r'(?:www\.)?invidious\.drycat\.fr',
 334         r'(?:www\.)?inv\.skyn3t\.in',
 335         r'(?:www\.)?tube\.poal\.co',
 336         r'(?:www\.)?tube\.connect\.cafe',
 337         r'(?:www\.)?vid\.wxzm\.sx',
 338         r'(?:www\.)?vid\.mint\.lgbt',
 339         r'(?:www\.)?vid\.puffyan\.us',
 340         r'(?:www\.)?yewtu\.be',
 341         r'(?:www\.)?yt\.elukerio\.org',
 342         r'(?:www\.)?yt\.lelux\.fi',
 343         r'(?:www\.)?invidious\.ggc-project\.de',
 344         r'(?:www\.)?yt\.maisputain\.ovh',
 345         r'(?:www\.)?ytprivate\.com',
 346         r'(?:www\.)?invidious\.13ad\.de',
 347         r'(?:www\.)?invidious\.toot\.koeln',
 348         r'(?:www\.)?invidious\.fdn\.fr',
 349         r'(?:www\.)?watch\.nettohikari\.com',
 350         r'(?:www\.)?invidious\.namazso\.eu',
 351         r'(?:www\.)?invidious\.silkky\.cloud',
 352         r'(?:www\.)?invidious\.exonip\.de',
 353         r'(?:www\.)?invidious\.riverside\.rocks',
 354         r'(?:www\.)?invidious\.blamefran\.net',
 355         r'(?:www\.)?invidious\.moomoo\.de',
 356         r'(?:www\.)?ytb\.trom\.tf',
 357         r'(?:www\.)?yt\.cyberhost\.uk',
 358         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 359         r'(?:www\.)?qklhadlycap4cnod\.onion',
 360         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 361         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 362         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 363         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 364         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 365         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 366         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 367         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 368         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 369         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 370         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 371         r'(?:www\.)?piped\.kavin\.rocks',
 372         r'(?:www\.)?piped\.tokhmi\.xyz',
 373         r'(?:www\.)?piped\.syncpundit\.io',
 374         r'(?:www\.)?piped\.mha\.fi',
 375         r'(?:www\.)?watch\.whatever\.social',
 376         r'(?:www\.)?piped\.garudalinux\.org',
 377         r'(?:www\.)?piped\.rivo\.lol',
 378         r'(?:www\.)?piped-libre\.kavin\.rocks',
 379         r'(?:www\.)?yt\.jae\.fi',
 380         r'(?:www\.)?piped\.mint\.lgbt',
 381         r'(?:www\.)?il\.ax',
 382         r'(?:www\.)?piped\.esmailelbob\.xyz',
 383         r'(?:www\.)?piped\.projectsegfau\.lt',
 384         r'(?:www\.)?piped\.privacydev\.net',
 385         r'(?:www\.)?piped\.palveluntarjoaja\.eu',
 386         r'(?:www\.)?piped\.smnz\.de',
 387         r'(?:www\.)?piped\.adminforge\.de',
 388         r'(?:www\.)?watch\.whatevertinfoil\.de',
 389         r'(?:www\.)?piped\.qdi\.fi',
 390     )
 391
 392     # extracted from account/account_menu ep
 393     # XXX: These are the supported YouTube UI and API languages,
 394     # which is slightly different from languages supported for translation in YouTube studio
 395     _SUPPORTED_LANG_CODES = [
 396         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 397         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 398         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 399         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 400         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 401         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 402     ]
 403
 404     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 405
 406     @functools.cached_property
 407     def _preferred_lang(self):
 408         """
 409         Returns a language code supported by YouTube for the user preferred language.
 410         Returns None if no preferred language set.
 411         """
 412         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 413         if not preferred_lang:
 414             return
 415         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 416             raise ExtractorError(
 417                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 418                 expected=True)
 419         elif preferred_lang != 'en':
 420             self.report_warning(
 421                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 422         return preferred_lang
 423
 424     def _initialize_consent(self):
 425         cookies = self._get_cookies('https://www.youtube.com/')
 426         if cookies.get('__Secure-3PSID'):
 427             return
 428         consent_id = None
 429         consent = cookies.get('CONSENT')
 430         if consent:
 431             if 'YES' in consent.value:
 432                 return
 433             consent_id = self._search_regex(
 434                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 435         if not consent_id:
 436             consent_id = random.randint(100, 999)
 437         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 438
 439     def _initialize_pref(self):
 440         cookies = self._get_cookies('https://www.youtube.com/')
 441         pref_cookie = cookies.get('PREF')
 442         pref = {}
 443         if pref_cookie:
 444             try:
 445                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 446             except ValueError:
 447                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 448         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 449         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 450
 451     def _real_initialize(self):
 452         self._initialize_pref()
 453         self._initialize_consent()
 454         self._check_login_required()
 455
 456     def _check_login_required(self):
 457         if self._LOGIN_REQUIRED and not self._cookies_passed:
 458             self.raise_login_required('Login details are needed to download this content', method='cookies')
 459
 460     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 461     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 462
 463     def _get_default_ytcfg(self, client='web'):
 464         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 465
 466     def _get_innertube_host(self, client='web'):
 467         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 468
 469     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 470         # try_get but with fallback to default ytcfg client values when present
 471         _func = lambda y: try_get(y, getter, expected_type)
 472         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 473
 474     def _extract_client_name(self, ytcfg, default_client='web'):
 475         return self._ytcfg_get_safe(
 476             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 477                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 478
 479     def _extract_client_version(self, ytcfg, default_client='web'):
 480         return self._ytcfg_get_safe(
 481             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 482                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 483
 484     def _select_api_hostname(self, req_api_hostname, default_client=None):
 485         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 486                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 487
 488     def _extract_api_key(self, ytcfg=None, default_client='web'):
 489         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 490
 491     def _extract_context(self, ytcfg=None, default_client='web'):
 492         context = get_first(
 493             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 494         # Enforce language and tz for extraction
 495         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 496         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 497         return context
 498
 499     _SAPISID = None
 500
 501     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 502         time_now = round(time.time())
 503         if self._SAPISID is None:
 504             yt_cookies = self._get_cookies('https://www.youtube.com')
 505             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 506             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 507             sapisid_cookie = dict_get(
 508                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 509             if sapisid_cookie and sapisid_cookie.value:
 510                 self._SAPISID = sapisid_cookie.value
 511                 self.write_debug('Extracted SAPISID cookie')
 512                 # SAPISID cookie is required if not already present
 513                 if not yt_cookies.get('SAPISID'):
 514                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 515                     self._set_cookie(
 516                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 517             else:
 518                 self._SAPISID = False
 519         if not self._SAPISID:
 520             return None
 521         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 522         sapisidhash = hashlib.sha1(
 523             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 524         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 525
 526     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 527                   note='Downloading API JSON', errnote='Unable to download API page',
 528                   context=None, api_key=None, api_hostname=None, default_client='web'):
 529
 530         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 531         data.update(query)
 532         real_headers = self.generate_api_headers(default_client=default_client)
 533         real_headers.update({'content-type': 'application/json'})
 534         if headers:
 535             real_headers.update(headers)
 536         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 537                    or api_key or self._extract_api_key(default_client=default_client))
 538         return self._download_json(
 539             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 540             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 541             data=json.dumps(data).encode('utf8'), headers=real_headers,
 542             query={'key': api_key, 'prettyPrint': 'false'})
 543
 544     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 545         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 546
 547     @staticmethod
 548     def _extract_session_index(*data):
 549         """
 550         Index of current account in account list.
 551         See: https://github.com/yt-dlp/yt-dlp/pull/519
 552         """
 553         for ytcfg in data:
 554             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 555             if session_index is not None:
 556                 return session_index
 557
 558     # Deprecated?
 559     def _extract_identity_token(self, ytcfg=None, webpage=None):
 560         if ytcfg:
 561             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 562             if token:
 563                 return token
 564         if webpage:
 565             return self._search_regex(
 566                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 567                 'identity token', default=None, fatal=False)
 568
 569     @staticmethod
 570     def _extract_account_syncid(*args):
 571         """
 572         Extract syncId required to download private playlists of secondary channels
 573         @params response and/or ytcfg
 574         """
 575         for data in args:
 576             # ytcfg includes channel_syncid if on secondary channel
 577             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 578             if delegated_sid:
 579                 return delegated_sid
 580             sync_ids = (try_get(
 581                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 582                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 583             if len(sync_ids) >= 2 and sync_ids[1]:
 584                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 585                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 586                 return sync_ids[0]
 587
 588     @staticmethod
 589     def _extract_visitor_data(*args):
 590         """
 591         Extracts visitorData from an API response or ytcfg
 592         Appears to be used to track session state
 593         """
 594         return get_first(
 595             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 596             expected_type=str)
 597
 598     @functools.cached_property
 599     def is_authenticated(self):
 600         return bool(self._generate_sapisidhash_header())
 601
 602     def extract_ytcfg(self, video_id, webpage):
 603         if not webpage:
 604             return {}
 605         return self._parse_json(
 606             self._search_regex(
 607                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 608                 default='{}'), video_id, fatal=False) or {}
 609
 610     def generate_api_headers(
 611             self, *, ytcfg=None, account_syncid=None, session_index=None,
 612             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 613
 614         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 615         headers = {
 616             'X-YouTube-Client-Name': str(
 617                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 618             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 619             'Origin': origin,
 620             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 621             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 622             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 623             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 624         }
 625         if session_index is None:
 626             session_index = self._extract_session_index(ytcfg)
 627         if account_syncid or session_index is not None:
 628             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 629
 630         auth = self._generate_sapisidhash_header(origin)
 631         if auth is not None:
 632             headers['Authorization'] = auth
 633             headers['X-Origin'] = origin
 634         return filter_dict(headers)
 635
 636     def _download_ytcfg(self, client, video_id):
 637         url = {
 638             'web': 'https://www.youtube.com',
 639             'web_music': 'https://music.youtube.com',
 640             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 641         }.get(client)
 642         if not url:
 643             return {}
 644         webpage = self._download_webpage(
 645             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 646         return self.extract_ytcfg(video_id, webpage) or {}
 647
 648     @staticmethod
 649     def _build_api_continuation_query(continuation, ctp=None):
 650         query = {
 651             'continuation': continuation
 652         }
 653         # TODO: Inconsistency with clickTrackingParams.
 654         # Currently we have a fixed ctp contained within context (from ytcfg)
 655         # and a ctp in root query for continuation.
 656         if ctp:
 657             query['clickTracking'] = {'clickTrackingParams': ctp}
 658         return query
 659
 660     @classmethod
 661     def _extract_next_continuation_data(cls, renderer):
 662         next_continuation = try_get(
 663             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 664                        lambda x: x['continuation']['reloadContinuationData']), dict)
 665         if not next_continuation:
 666             return
 667         continuation = next_continuation.get('continuation')
 668         if not continuation:
 669             return
 670         ctp = next_continuation.get('clickTrackingParams')
 671         return cls._build_api_continuation_query(continuation, ctp)
 672
 673     @classmethod
 674     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 675         if isinstance(continuation_ep, dict):
 676             continuation = try_get(
 677                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 678             if not continuation:
 679                 return
 680             ctp = continuation_ep.get('clickTrackingParams')
 681             return cls._build_api_continuation_query(continuation, ctp)
 682
 683     @classmethod
 684     def _extract_continuation(cls, renderer):
 685         next_continuation = cls._extract_next_continuation_data(renderer)
 686         if next_continuation:
 687             return next_continuation
 688
 689         return traverse_obj(renderer, (
 690             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 691             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 692         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 693
 694     @classmethod
 695     def _extract_alerts(cls, data):
 696         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 697             if not isinstance(alert_dict, dict):
 698                 continue
 699             for alert in alert_dict.values():
 700                 alert_type = alert.get('type')
 701                 if not alert_type:
 702                     continue
 703                 message = cls._get_text(alert, 'text')
 704                 if message:
 705                     yield alert_type, message
 706
 707     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 708         errors, warnings = [], []
 709         for alert_type, alert_message in alerts:
 710             if alert_type.lower() == 'error' and fatal:
 711                 errors.append([alert_type, alert_message])
 712             elif alert_message not in self._IGNORED_WARNINGS:
 713                 warnings.append([alert_type, alert_message])
 714
 715         for alert_type, alert_message in (warnings + errors[:-1]):
 716             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 717         if errors:
 718             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 719
 720     def _extract_and_report_alerts(self, data, *args, **kwargs):
 721         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 722
 723     def _extract_badges(self, renderer: dict):
 724         privacy_icon_map = {
 725             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 726             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 727             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 728         }
 729
 730         badge_style_map = {
 731             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 732             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 733             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 734         }
 735
 736         label_map = {
 737             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 738             'private': BadgeType.AVAILABILITY_PRIVATE,
 739             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 740             'live': BadgeType.LIVE_NOW,
 741             'premium': BadgeType.AVAILABILITY_PREMIUM
 742         }
 743
 744         badges = []
 745         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer'), default=[]):
 746             badge_type = (
 747                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 748                 or badge_style_map.get(traverse_obj(badge, 'style'))
 749             )
 750             if badge_type:
 751                 badges.append({'type': badge_type})
 752                 continue
 753
 754             # fallback, won't work in some languages
 755             label = traverse_obj(badge, 'label', expected_type=str, default='')
 756             for match, label_badge_type in label_map.items():
 757                 if match in label.lower():
 758                     badges.append({'type': badge_type})
 759                     continue
 760
 761         return badges
 762
 763     @staticmethod
 764     def _has_badge(badges, badge_type):
 765         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 766
 767     @staticmethod
 768     def _get_text(data, *path_list, max_runs=None):
 769         for path in path_list or [None]:
 770             if path is None:
 771                 obj = [data]
 772             else:
 773                 obj = traverse_obj(data, path, default=[])
 774                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 775                     obj = [obj]
 776             for item in obj:
 777                 text = try_get(item, lambda x: x['simpleText'], str)
 778                 if text:
 779                     return text
 780                 runs = try_get(item, lambda x: x['runs'], list) or []
 781                 if not runs and isinstance(item, list):
 782                     runs = item
 783
 784                 runs = runs[:min(len(runs), max_runs or len(runs))]
 785                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 786                 if text:
 787                     return text
 788
 789     def _get_count(self, data, *path_list):
 790         count_text = self._get_text(data, *path_list) or ''
 791         count = parse_count(count_text)
 792         if count is None:
 793             count = str_to_int(
 794                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 795         return count
 796
 797     @staticmethod
 798     def _extract_thumbnails(data, *path_list):
 799         """
 800         Extract thumbnails from thumbnails dict
 801         @param path_list: path list to level that contains 'thumbnails' key
 802         """
 803         thumbnails = []
 804         for path in path_list or [()]:
 805             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 806                 thumbnail_url = url_or_none(thumbnail.get('url'))
 807                 if not thumbnail_url:
 808                     continue
 809                 # Sometimes youtube gives a wrong thumbnail URL. See:
 810                 # https://github.com/yt-dlp/yt-dlp/issues/233
 811                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 812                 if 'maxresdefault' in thumbnail_url:
 813                     thumbnail_url = thumbnail_url.split('?')[0]
 814                 thumbnails.append({
 815                     'url': thumbnail_url,
 816                     'height': int_or_none(thumbnail.get('height')),
 817                     'width': int_or_none(thumbnail.get('width')),
 818                 })
 819         return thumbnails
 820
 821     @staticmethod
 822     def extract_relative_time(relative_time_text):
 823         """
 824         Extracts a relative time from string and converts to dt object
 825         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 826         """
 827         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 828         if mobj:
 829             start = mobj.group('start')
 830             if start:
 831                 return datetime_from_str(start)
 832             try:
 833                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 834             except ValueError:
 835                 return None
 836
 837     def _parse_time_text(self, text):
 838         if not text:
 839             return
 840         dt = self.extract_relative_time(text)
 841         timestamp = None
 842         if isinstance(dt, datetime.datetime):
 843             timestamp = calendar.timegm(dt.timetuple())
 844
 845         if timestamp is None:
 846             timestamp = (
 847                 unified_timestamp(text) or unified_timestamp(
 848                     self._search_regex(
 849                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 850                         text.lower(), 'time text', default=None)))
 851
 852         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 853             self.report_warning(
 854                 f'Cannot parse localized time text "{text}"', only_once=True)
 855         return timestamp
 856
 857     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 858                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 859                           default_client='web'):
 860         for retry in self.RetryManager():
 861             try:
 862                 response = self._call_api(
 863                     ep=ep, fatal=True, headers=headers,
 864                     video_id=item_id, query=query, note=note,
 865                     context=self._extract_context(ytcfg, default_client),
 866                     api_key=self._extract_api_key(ytcfg, default_client),
 867                     api_hostname=api_hostname, default_client=default_client)
 868             except ExtractorError as e:
 869                 if not isinstance(e.cause, network_exceptions):
 870                     return self._error_or_warning(e, fatal=fatal)
 871                 elif not isinstance(e.cause, urllib.error.HTTPError):
 872                     retry.error = e
 873                     continue
 874
 875                 first_bytes = e.cause.read(512)
 876                 if not is_html(first_bytes):
 877                     yt_error = try_get(
 878                         self._parse_json(
 879                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 880                         lambda x: x['error']['message'], str)
 881                     if yt_error:
 882                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 883                 # Downloading page may result in intermittent 5xx HTTP error
 884                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 885                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 886                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 887                 if e.cause.code not in (403, 429):
 888                     retry.error = e
 889                     continue
 890                 return self._error_or_warning(e, fatal=fatal)
 891
 892             try:
 893                 self._extract_and_report_alerts(response, only_once=True)
 894             except ExtractorError as e:
 895                 # YouTube servers may return errors we want to retry on in a 200 OK response
 896                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 897                 if 'unknown error' in e.msg.lower():
 898                     retry.error = e
 899                     continue
 900                 return self._error_or_warning(e, fatal=fatal)
 901             # Youtube sometimes sends incomplete data
 902             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 903             if not traverse_obj(response, *variadic(check_get_keys)):
 904                 retry.error = ExtractorError('Incomplete data received', expected=True)
 905                 continue
 906
 907             return response
 908
 909     @staticmethod
 910     def is_music_url(url):
 911         return re.match(r'https?://music\.youtube\.com/', url) is not None
 912
 913     def _extract_video(self, renderer):
 914         video_id = renderer.get('videoId')
 915
 916         reel_header_renderer = traverse_obj(renderer, (
 917             'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
 918             'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
 919
 920         title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
 921         description = self._get_text(renderer, 'descriptionSnippet')
 922
 923         duration = int_or_none(renderer.get('lengthSeconds'))
 924         if duration is None:
 925             duration = parse_duration(self._get_text(
 926                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 927         if duration is None:
 928             # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
 929             duration = parse_duration(self._search_regex(
 930                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 931                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 932                 video_id, default=None, group='duration'))
 933
 934         channel_id = traverse_obj(
 935             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 936             expected_type=str, get_all=False)
 937         if not channel_id:
 938             channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
 939
 940         overlay_style = traverse_obj(
 941             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 942             get_all=False, expected_type=str)
 943         badges = self._extract_badges(renderer)
 944
 945         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 946             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 947             expected_type=str)) or ''
 948         url = f'https://www.youtube.com/watch?v={video_id}'
 949         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 950             url = f'https://www.youtube.com/shorts/{video_id}'
 951
 952         time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
 953                      or self._get_text(reel_header_renderer, 'timestampText') or '')
 954         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 955
 956         live_status = (
 957             'is_upcoming' if scheduled_timestamp is not None
 958             else 'was_live' if 'streamed' in time_text.lower()
 959             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
 960             else None)
 961
 962         # videoInfo is a string like '50K views • 10 years ago'.
 963         view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
 964         view_count = (0 if 'no views' in view_count_text.lower()
 965                       else self._get_count({'simpleText': view_count_text}))
 966         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
 967
 968         return {
 969             '_type': 'url',
 970             'ie_key': YoutubeIE.ie_key(),
 971             'id': video_id,
 972             'url': url,
 973             'title': title,
 974             'description': description,
 975             'duration': duration,
 976             'channel_id': channel_id,
 977             'channel': (self._get_text(renderer, 'ownerText', 'shortBylineText')
 978                         or self._get_text(reel_header_renderer, 'channelTitleText')),
 979             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
 980             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
 981             'timestamp': (self._parse_time_text(time_text)
 982                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
 983                           else None),
 984             'release_timestamp': scheduled_timestamp,
 985             'availability':
 986                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
 987                 else self._availability(
 988                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
 989                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
 990                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
 991                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
 992             view_count_field: view_count,
 993             'live_status': live_status
 994         }
 995
 996
 997 class YoutubeIE(YoutubeBaseInfoExtractor):
 998     IE_DESC = 'YouTube'
 999     _VALID_URL = r"""(?x)^
1000                      (
1001                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
1002                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
1003                             (?:www\.)?deturl\.com/www\.youtube\.com|
1004                             (?:www\.)?pwnyoutube\.com|
1005                             (?:www\.)?hooktube\.com|
1006                             (?:www\.)?yourepeat\.com|
1007                             tube\.majestyc\.net|
1008                             %(invidious)s|
1009                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
1010                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
1011                          (?:                                                  # the various things that can precede the ID:
1012                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
1013                              |(?:                                             # or the v= param in all its forms
1014                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
1015                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
1016                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
1017                                  v=
1018                              )
1019                          ))
1020                          |(?:
1021                             youtu\.be|                                        # just youtu.be/xxxx
1022                             vid\.plus|                                        # or vid.plus/xxxx
1023                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
1024                             %(invidious)s
1025                          )/
1026                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1027                          )
1028                      )?                                                       # all until now is optional -> you can pass the naked ID
1029                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1030                      (?(1).+)?                                                # if we found the ID, everything can follow
1031                      (?:\#|$)""" % {
1032         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1033     }
1034     _EMBED_REGEX = [
1035         r'''(?x)
1036             (?:
1037                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1038                 data-video-url=|
1039                 <embed[^>]+?src=|
1040                 embedSWF\(?:\s*|
1041                 <object[^>]+data=|
1042                 new\s+SWFObject\(
1043             )
1044             (["\'])
1045                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1046                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1047             \1''',
1048         # https://wordpress.org/plugins/lazy-load-for-videos/
1049         r'''(?xs)
1050             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1051             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1052     ]
1053
1054     _PLAYER_INFO_RE = (
1055         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1056         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1057         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1058     )
1059     _formats = {
1060         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1061         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1062         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1063         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1064         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1065         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1066         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1067         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1068         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1069         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1070         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1071         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1072         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1073         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1074         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1075         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1076         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1077         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1078
1079
1080         # 3D videos
1081         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1082         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1083         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1084         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1085         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1086         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1087         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1088
1089         # Apple HTTP Live Streaming
1090         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1091         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1092         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1093         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1094         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1095         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1096         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1097         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1098
1099         # DASH mp4 video
1100         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1101         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1102         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1103         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1104         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1105         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1106         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1107         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1108         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1109         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1110         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1111         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1112
1113         # Dash mp4 audio
1114         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1115         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1116         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1117         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1118         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1119         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1120         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1121
1122         # Dash webm
1123         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1124         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1125         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1126         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1127         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1128         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1129         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1130         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1131         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1132         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1133         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1134         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1135         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1136         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1137         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1138         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1139         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1140         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1141         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1142         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1143         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1144         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1145
1146         # Dash webm audio
1147         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1148         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1149
1150         # Dash webm audio with opus inside
1151         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1152         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1153         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1154
1155         # RTMP (unnamed)
1156         '_rtmp': {'protocol': 'rtmp'},
1157
1158         # av01 video only formats sometimes served with "unknown" codecs
1159         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1160         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1161         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1162         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1163         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1164         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1165         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1166         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1167     }
1168     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1169
1170     _GEO_BYPASS = False
1171
1172     IE_NAME = 'youtube'
1173     _TESTS = [
1174         {
1175             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1176             'info_dict': {
1177                 'id': 'BaW_jenozKc',
1178                 'ext': 'mp4',
1179                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1180                 'uploader': 'Philipp Hagemeister',
1181                 'uploader_id': 'phihag',
1182                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1183                 'channel': 'Philipp Hagemeister',
1184                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1185                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1186                 'upload_date': '20121002',
1187                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1188                 'categories': ['Science & Technology'],
1189                 'tags': ['youtube-dl'],
1190                 'duration': 10,
1191                 'view_count': int,
1192                 'like_count': int,
1193                 'availability': 'public',
1194                 'playable_in_embed': True,
1195                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1196                 'live_status': 'not_live',
1197                 'age_limit': 0,
1198                 'start_time': 1,
1199                 'end_time': 9,
1200                 'comment_count': int,
1201                 'channel_follower_count': int
1202             }
1203         },
1204         {
1205             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1206             'note': 'Embed-only video (#1746)',
1207             'info_dict': {
1208                 'id': 'yZIXLfi8CZQ',
1209                 'ext': 'mp4',
1210                 'upload_date': '20120608',
1211                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1212                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1213                 'uploader': 'SET India',
1214                 'uploader_id': 'setindia',
1215                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1216                 'age_limit': 18,
1217             },
1218             'skip': 'Private video',
1219         },
1220         {
1221             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1222             'note': 'Use the first video ID in the URL',
1223             'info_dict': {
1224                 'id': 'BaW_jenozKc',
1225                 'ext': 'mp4',
1226                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1227                 'uploader': 'Philipp Hagemeister',
1228                 'uploader_id': 'phihag',
1229                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1230                 'channel': 'Philipp Hagemeister',
1231                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1232                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1233                 'upload_date': '20121002',
1234                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1235                 'categories': ['Science & Technology'],
1236                 'tags': ['youtube-dl'],
1237                 'duration': 10,
1238                 'view_count': int,
1239                 'like_count': int,
1240                 'availability': 'public',
1241                 'playable_in_embed': True,
1242                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1243                 'live_status': 'not_live',
1244                 'age_limit': 0,
1245                 'comment_count': int,
1246                 'channel_follower_count': int
1247             },
1248             'params': {
1249                 'skip_download': True,
1250             },
1251         },
1252         {
1253             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1254             'note': '256k DASH audio (format 141) via DASH manifest',
1255             'info_dict': {
1256                 'id': 'a9LDPn-MO4I',
1257                 'ext': 'm4a',
1258                 'upload_date': '20121002',
1259                 'uploader_id': '8KVIDEO',
1260                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1261                 'description': '',
1262                 'uploader': '8KVIDEO',
1263                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1264             },
1265             'params': {
1266                 'youtube_include_dash_manifest': True,
1267                 'format': '141',
1268             },
1269             'skip': 'format 141 not served anymore',
1270         },
1271         # DASH manifest with encrypted signature
1272         {
1273             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1274             'info_dict': {
1275                 'id': 'IB3lcPjvWLA',
1276                 'ext': 'm4a',
1277                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1278                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1279                 'duration': 244,
1280                 'uploader': 'AfrojackVEVO',
1281                 'uploader_id': 'AfrojackVEVO',
1282                 'upload_date': '20131011',
1283                 'abr': 129.495,
1284                 'like_count': int,
1285                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1286                 'playable_in_embed': True,
1287                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1288                 'view_count': int,
1289                 'track': 'The Spark',
1290                 'live_status': 'not_live',
1291                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1292                 'channel': 'Afrojack',
1293                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1294                 'tags': 'count:19',
1295                 'availability': 'public',
1296                 'categories': ['Music'],
1297                 'age_limit': 0,
1298                 'alt_title': 'The Spark',
1299                 'channel_follower_count': int
1300             },
1301             'params': {
1302                 'youtube_include_dash_manifest': True,
1303                 'format': '141/bestaudio[ext=m4a]',
1304             },
1305         },
1306         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1307         {
1308             'note': 'Embed allowed age-gate video',
1309             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1310             'info_dict': {
1311                 'id': 'HtVdAasjOgU',
1312                 'ext': 'mp4',
1313                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1314                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1315                 'duration': 142,
1316                 'uploader': 'The Witcher',
1317                 'uploader_id': 'WitcherGame',
1318                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1319                 'upload_date': '20140605',
1320                 'age_limit': 18,
1321                 'categories': ['Gaming'],
1322                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1323                 'availability': 'needs_auth',
1324                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1325                 'like_count': int,
1326                 'channel': 'The Witcher',
1327                 'live_status': 'not_live',
1328                 'tags': 'count:17',
1329                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1330                 'playable_in_embed': True,
1331                 'view_count': int,
1332                 'channel_follower_count': int
1333             },
1334         },
1335         {
1336             'note': 'Age-gate video with embed allowed in public site',
1337             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1338             'info_dict': {
1339                 'id': 'HsUATh_Nc2U',
1340                 'ext': 'mp4',
1341                 'title': 'Godzilla 2 (Official Video)',
1342                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1343                 'upload_date': '20200408',
1344                 'uploader_id': 'FlyingKitty900',
1345                 'uploader': 'FlyingKitty',
1346                 'age_limit': 18,
1347                 'availability': 'needs_auth',
1348                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1349                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1350                 'channel': 'FlyingKitty',
1351                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1352                 'view_count': int,
1353                 'categories': ['Entertainment'],
1354                 'live_status': 'not_live',
1355                 'tags': ['Flyingkitty', 'godzilla 2'],
1356                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1357                 'like_count': int,
1358                 'duration': 177,
1359                 'playable_in_embed': True,
1360                 'channel_follower_count': int
1361             },
1362         },
1363         {
1364             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1365             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1366             'info_dict': {
1367                 'id': 'Tq92D6wQ1mg',
1368                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1369                 'ext': 'mp4',
1370                 'upload_date': '20191228',
1371                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1372                 'uploader': 'Projekt Melody',
1373                 'description': 'md5:17eccca93a786d51bc67646756894066',
1374                 'age_limit': 18,
1375                 'like_count': int,
1376                 'availability': 'needs_auth',
1377                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1378                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1379                 'view_count': int,
1380                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1381                 'channel': 'Projekt Melody',
1382                 'live_status': 'not_live',
1383                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1384                 'playable_in_embed': True,
1385                 'categories': ['Entertainment'],
1386                 'duration': 106,
1387                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1388                 'comment_count': int,
1389                 'channel_follower_count': int
1390             },
1391         },
1392         {
1393             'note': 'Non-Agegated non-embeddable video',
1394             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1395             'info_dict': {
1396                 'id': 'MeJVWBSsPAY',
1397                 'ext': 'mp4',
1398                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1399                 'uploader': 'Herr Lurik',
1400                 'uploader_id': 'st3in234',
1401                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1402                 'upload_date': '20130730',
1403                 'track': 'Such mich find mich',
1404                 'age_limit': 0,
1405                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1406                 'like_count': int,
1407                 'playable_in_embed': False,
1408                 'creator': 'OOMPH!',
1409                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1410                 'view_count': int,
1411                 'alt_title': 'Such mich find mich',
1412                 'duration': 210,
1413                 'channel': 'Herr Lurik',
1414                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1415                 'categories': ['Music'],
1416                 'availability': 'public',
1417                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1418                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1419                 'live_status': 'not_live',
1420                 'artist': 'OOMPH!',
1421                 'channel_follower_count': int
1422             },
1423         },
1424         {
1425             'note': 'Non-bypassable age-gated video',
1426             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1427             'only_matching': True,
1428         },
1429         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1430         # YouTube Red ad is not captured for creator
1431         {
1432             'url': '__2ABJjxzNo',
1433             'info_dict': {
1434                 'id': '__2ABJjxzNo',
1435                 'ext': 'mp4',
1436                 'duration': 266,
1437                 'upload_date': '20100430',
1438                 'uploader_id': 'deadmau5',
1439                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1440                 'creator': 'deadmau5',
1441                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1442                 'uploader': 'deadmau5',
1443                 'title': 'Deadmau5 - Some Chords (HD)',
1444                 'alt_title': 'Some Chords',
1445                 'availability': 'public',
1446                 'tags': 'count:14',
1447                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1448                 'view_count': int,
1449                 'live_status': 'not_live',
1450                 'channel': 'deadmau5',
1451                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1452                 'like_count': int,
1453                 'track': 'Some Chords',
1454                 'artist': 'deadmau5',
1455                 'playable_in_embed': True,
1456                 'age_limit': 0,
1457                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1458                 'categories': ['Music'],
1459                 'album': 'Some Chords',
1460                 'channel_follower_count': int
1461             },
1462             'expected_warnings': [
1463                 'DASH manifest missing',
1464             ]
1465         },
1466         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1467         {
1468             'url': 'lqQg6PlCWgI',
1469             'info_dict': {
1470                 'id': 'lqQg6PlCWgI',
1471                 'ext': 'mp4',
1472                 'duration': 6085,
1473                 'upload_date': '20150827',
1474                 'uploader_id': 'olympic',
1475                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1476                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1477                 'uploader': 'Olympics',
1478                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1479                 'like_count': int,
1480                 'release_timestamp': 1343767800,
1481                 'playable_in_embed': True,
1482                 'categories': ['Sports'],
1483                 'release_date': '20120731',
1484                 'channel': 'Olympics',
1485                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1486                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1487                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1488                 'age_limit': 0,
1489                 'availability': 'public',
1490                 'live_status': 'was_live',
1491                 'view_count': int,
1492                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1493                 'channel_follower_count': int
1494             },
1495             'params': {
1496                 'skip_download': 'requires avconv',
1497             }
1498         },
1499         # Non-square pixels
1500         {
1501             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1502             'info_dict': {
1503                 'id': '_b-2C3KPAM0',
1504                 'ext': 'mp4',
1505                 'stretched_ratio': 16 / 9.,
1506                 'duration': 85,
1507                 'upload_date': '20110310',
1508                 'uploader_id': 'AllenMeow',
1509                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1510                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1511                 'uploader': '孫ᄋᄅ',
1512                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1513                 'playable_in_embed': True,
1514                 'channel': '孫ᄋᄅ',
1515                 'age_limit': 0,
1516                 'tags': 'count:11',
1517                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1518                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1519                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1520                 'view_count': int,
1521                 'categories': ['People & Blogs'],
1522                 'like_count': int,
1523                 'live_status': 'not_live',
1524                 'availability': 'unlisted',
1525                 'comment_count': int,
1526                 'channel_follower_count': int
1527             },
1528         },
1529         # url_encoded_fmt_stream_map is empty string
1530         {
1531             'url': 'qEJwOuvDf7I',
1532             'info_dict': {
1533                 'id': 'qEJwOuvDf7I',
1534                 'ext': 'webm',
1535                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1536                 'description': '',
1537                 'upload_date': '20150404',
1538                 'uploader_id': 'spbelect',
1539                 'uploader': 'Наблюдатели Петербурга',
1540             },
1541             'params': {
1542                 'skip_download': 'requires avconv',
1543             },
1544             'skip': 'This live event has ended.',
1545         },
1546         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1547         {
1548             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1549             'info_dict': {
1550                 'id': 'FIl7x6_3R5Y',
1551                 'ext': 'webm',
1552                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1553                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1554                 'duration': 220,
1555                 'upload_date': '20150625',
1556                 'uploader_id': 'dorappi2000',
1557                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1558                 'uploader': 'dorappi2000',
1559                 'formats': 'mincount:31',
1560             },
1561             'skip': 'not actual anymore',
1562         },
1563         # DASH manifest with segment_list
1564         {
1565             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1566             'md5': '8ce563a1d667b599d21064e982ab9e31',
1567             'info_dict': {
1568                 'id': 'CsmdDsKjzN8',
1569                 'ext': 'mp4',
1570                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1571                 'uploader': 'Airtek',
1572                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1573                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1574                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1575             },
1576             'params': {
1577                 'youtube_include_dash_manifest': True,
1578                 'format': '135',  # bestvideo
1579             },
1580             'skip': 'This live event has ended.',
1581         },
1582         {
1583             # Multifeed videos (multiple cameras), URL is for Main Camera
1584             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1585             'info_dict': {
1586                 'id': 'jvGDaLqkpTg',
1587                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1588                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1589             },
1590             'playlist': [{
1591                 'info_dict': {
1592                     'id': 'jvGDaLqkpTg',
1593                     'ext': 'mp4',
1594                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1595                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1596                     'duration': 10643,
1597                     'upload_date': '20161111',
1598                     'uploader': 'Team PGP',
1599                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1600                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1601                 },
1602             }, {
1603                 'info_dict': {
1604                     'id': '3AKt1R1aDnw',
1605                     'ext': 'mp4',
1606                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1607                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1608                     'duration': 10991,
1609                     'upload_date': '20161111',
1610                     'uploader': 'Team PGP',
1611                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1612                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1613                 },
1614             }, {
1615                 'info_dict': {
1616                     'id': 'RtAMM00gpVc',
1617                     'ext': 'mp4',
1618                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1619                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1620                     'duration': 10995,
1621                     'upload_date': '20161111',
1622                     'uploader': 'Team PGP',
1623                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1624                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1625                 },
1626             }, {
1627                 'info_dict': {
1628                     'id': '6N2fdlP3C5U',
1629                     'ext': 'mp4',
1630                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1631                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1632                     'duration': 10990,
1633                     'upload_date': '20161111',
1634                     'uploader': 'Team PGP',
1635                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1636                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1637                 },
1638             }],
1639             'params': {
1640                 'skip_download': True,
1641             },
1642             'skip': 'Not multifeed anymore',
1643         },
1644         {
1645             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1646             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1647             'info_dict': {
1648                 'id': 'gVfLd0zydlo',
1649                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1650             },
1651             'playlist_count': 2,
1652             'skip': 'Not multifeed anymore',
1653         },
1654         {
1655             'url': 'https://vid.plus/FlRa-iH7PGw',
1656             'only_matching': True,
1657         },
1658         {
1659             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1660             'only_matching': True,
1661         },
1662         {
1663             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1664             # Also tests cut-off URL expansion in video description (see
1665             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1666             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1667             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1668             'info_dict': {
1669                 'id': 'lsguqyKfVQg',
1670                 'ext': 'mp4',
1671                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1672                 'alt_title': 'Dark Walk',
1673                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1674                 'duration': 133,
1675                 'upload_date': '20151119',
1676                 'uploader_id': 'IronSoulElf',
1677                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1678                 'uploader': 'IronSoulElf',
1679                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1680                 'track': 'Dark Walk',
1681                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1682                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1683                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1684                 'categories': ['Film & Animation'],
1685                 'view_count': int,
1686                 'live_status': 'not_live',
1687                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1688                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1689                 'tags': 'count:13',
1690                 'availability': 'public',
1691                 'channel': 'IronSoulElf',
1692                 'playable_in_embed': True,
1693                 'like_count': int,
1694                 'age_limit': 0,
1695                 'channel_follower_count': int
1696             },
1697             'params': {
1698                 'skip_download': True,
1699             },
1700         },
1701         {
1702             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1703             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1704             'only_matching': True,
1705         },
1706         {
1707             # Video with yt:stretch=17:0
1708             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1709             'info_dict': {
1710                 'id': 'Q39EVAstoRM',
1711                 'ext': 'mp4',
1712                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1713                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1714                 'upload_date': '20151107',
1715                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1716                 'uploader': 'CH GAMER DROID',
1717             },
1718             'params': {
1719                 'skip_download': True,
1720             },
1721             'skip': 'This video does not exist.',
1722         },
1723         {
1724             # Video with incomplete 'yt:stretch=16:'
1725             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1726             'only_matching': True,
1727         },
1728         {
1729             # Video licensed under Creative Commons
1730             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1731             'info_dict': {
1732                 'id': 'M4gD1WSo5mA',
1733                 'ext': 'mp4',
1734                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1735                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1736                 'duration': 721,
1737                 'upload_date': '20150128',
1738                 'uploader_id': 'BerkmanCenter',
1739                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1740                 'uploader': 'The Berkman Klein Center for Internet & Society',
1741                 'license': 'Creative Commons Attribution license (reuse allowed)',
1742                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1743                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1744                 'like_count': int,
1745                 'age_limit': 0,
1746                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1747                 'channel': 'The Berkman Klein Center for Internet & Society',
1748                 'availability': 'public',
1749                 'view_count': int,
1750                 'categories': ['Education'],
1751                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1752                 'live_status': 'not_live',
1753                 'playable_in_embed': True,
1754                 'comment_count': int,
1755                 'channel_follower_count': int,
1756                 'chapters': list,
1757             },
1758             'params': {
1759                 'skip_download': True,
1760             },
1761         },
1762         {
1763             # Channel-like uploader_url
1764             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1765             'info_dict': {
1766                 'id': 'eQcmzGIKrzg',
1767                 'ext': 'mp4',
1768                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1769                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1770                 'duration': 4060,
1771                 'upload_date': '20151120',
1772                 'uploader': 'Bernie Sanders',
1773                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1774                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1775                 'license': 'Creative Commons Attribution license (reuse allowed)',
1776                 'playable_in_embed': True,
1777                 'tags': 'count:12',
1778                 'like_count': int,
1779                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1780                 'age_limit': 0,
1781                 'availability': 'public',
1782                 'categories': ['News & Politics'],
1783                 'channel': 'Bernie Sanders',
1784                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1785                 'view_count': int,
1786                 'live_status': 'not_live',
1787                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1788                 'comment_count': int,
1789                 'channel_follower_count': int,
1790                 'chapters': list,
1791             },
1792             'params': {
1793                 'skip_download': True,
1794             },
1795         },
1796         {
1797             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1798             'only_matching': True,
1799         },
1800         {
1801             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1802             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1803             'only_matching': True,
1804         },
1805         {
1806             # Rental video preview
1807             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1808             'info_dict': {
1809                 'id': 'uGpuVWrhIzE',
1810                 'ext': 'mp4',
1811                 'title': 'Piku - Trailer',
1812                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1813                 'upload_date': '20150811',
1814                 'uploader': 'FlixMatrix',
1815                 'uploader_id': 'FlixMatrixKaravan',
1816                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1817                 'license': 'Standard YouTube License',
1818             },
1819             'params': {
1820                 'skip_download': True,
1821             },
1822             'skip': 'This video is not available.',
1823         },
1824         {
1825             # YouTube Red video with episode data
1826             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1827             'info_dict': {
1828                 'id': 'iqKdEhx-dD4',
1829                 'ext': 'mp4',
1830                 'title': 'Isolation - Mind Field (Ep 1)',
1831                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1832                 'duration': 2085,
1833                 'upload_date': '20170118',
1834                 'uploader': 'Vsauce',
1835                 'uploader_id': 'Vsauce',
1836                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1837                 'series': 'Mind Field',
1838                 'season_number': 1,
1839                 'episode_number': 1,
1840                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1841                 'tags': 'count:12',
1842                 'view_count': int,
1843                 'availability': 'public',
1844                 'age_limit': 0,
1845                 'channel': 'Vsauce',
1846                 'episode': 'Episode 1',
1847                 'categories': ['Entertainment'],
1848                 'season': 'Season 1',
1849                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1850                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1851                 'like_count': int,
1852                 'playable_in_embed': True,
1853                 'live_status': 'not_live',
1854                 'channel_follower_count': int
1855             },
1856             'params': {
1857                 'skip_download': True,
1858             },
1859             'expected_warnings': [
1860                 'Skipping DASH manifest',
1861             ],
1862         },
1863         {
1864             # The following content has been identified by the YouTube community
1865             # as inappropriate or offensive to some audiences.
1866             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1867             'info_dict': {
1868                 'id': '6SJNVb0GnPI',
1869                 'ext': 'mp4',
1870                 'title': 'Race Differences in Intelligence',
1871                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1872                 'duration': 965,
1873                 'upload_date': '20140124',
1874                 'uploader': 'New Century Foundation',
1875                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1876                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1877             },
1878             'params': {
1879                 'skip_download': True,
1880             },
1881             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1882         },
1883         {
1884             # itag 212
1885             'url': '1t24XAntNCY',
1886             'only_matching': True,
1887         },
1888         {
1889             # geo restricted to JP
1890             'url': 'sJL6WA-aGkQ',
1891             'only_matching': True,
1892         },
1893         {
1894             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1895             'only_matching': True,
1896         },
1897         {
1898             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1899             'only_matching': True,
1900         },
1901         {
1902             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1903             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1904             'only_matching': True,
1905         },
1906         {
1907             # DRM protected
1908             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1909             'only_matching': True,
1910         },
1911         {
1912             # Video with unsupported adaptive stream type formats
1913             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1914             'info_dict': {
1915                 'id': 'Z4Vy8R84T1U',
1916                 'ext': 'mp4',
1917                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1918                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1919                 'duration': 433,
1920                 'upload_date': '20130923',
1921                 'uploader': 'Amelia Putri Harwita',
1922                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1923                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1924                 'formats': 'maxcount:10',
1925             },
1926             'params': {
1927                 'skip_download': True,
1928                 'youtube_include_dash_manifest': False,
1929             },
1930             'skip': 'not actual anymore',
1931         },
1932         {
1933             # Youtube Music Auto-generated description
1934             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1935             'info_dict': {
1936                 'id': 'MgNrAu2pzNs',
1937                 'ext': 'mp4',
1938                 'title': 'Voyeur Girl',
1939                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1940                 'upload_date': '20190312',
1941                 'uploader': 'Stephen - Topic',
1942                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1943                 'artist': 'Stephen',
1944                 'track': 'Voyeur Girl',
1945                 'album': 'it\'s too much love to know my dear',
1946                 'release_date': '20190313',
1947                 'release_year': 2019,
1948                 'alt_title': 'Voyeur Girl',
1949                 'view_count': int,
1950                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1951                 'playable_in_embed': True,
1952                 'like_count': int,
1953                 'categories': ['Music'],
1954                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1955                 'channel': 'Stephen',
1956                 'availability': 'public',
1957                 'creator': 'Stephen',
1958                 'duration': 169,
1959                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1960                 'age_limit': 0,
1961                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1962                 'tags': 'count:11',
1963                 'live_status': 'not_live',
1964                 'channel_follower_count': int
1965             },
1966             'params': {
1967                 'skip_download': True,
1968             },
1969         },
1970         {
1971             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1972             'only_matching': True,
1973         },
1974         {
1975             # invalid -> valid video id redirection
1976             'url': 'DJztXj2GPfl',
1977             'info_dict': {
1978                 'id': 'DJztXj2GPfk',
1979                 'ext': 'mp4',
1980                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1981                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1982                 'upload_date': '20090125',
1983                 'uploader': 'Prochorowka',
1984                 'uploader_id': 'Prochorowka',
1985                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1986                 'artist': 'Panjabi MC',
1987                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1988                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1989             },
1990             'params': {
1991                 'skip_download': True,
1992             },
1993             'skip': 'Video unavailable',
1994         },
1995         {
1996             # empty description results in an empty string
1997             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1998             'info_dict': {
1999                 'id': 'x41yOUIvK2k',
2000                 'ext': 'mp4',
2001                 'title': 'IMG 3456',
2002                 'description': '',
2003                 'upload_date': '20170613',
2004                 'uploader_id': 'ElevageOrVert',
2005                 'uploader': 'ElevageOrVert',
2006                 'view_count': int,
2007                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
2008                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
2009                 'like_count': int,
2010                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
2011                 'tags': [],
2012                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
2013                 'availability': 'public',
2014                 'age_limit': 0,
2015                 'categories': ['Pets & Animals'],
2016                 'duration': 7,
2017                 'playable_in_embed': True,
2018                 'live_status': 'not_live',
2019                 'channel': 'ElevageOrVert',
2020                 'channel_follower_count': int
2021             },
2022             'params': {
2023                 'skip_download': True,
2024             },
2025         },
2026         {
2027             # with '};' inside yt initial data (see [1])
2028             # see [2] for an example with '};' inside ytInitialPlayerResponse
2029             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2030             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2031             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2032             'info_dict': {
2033                 'id': 'CHqg6qOn4no',
2034                 'ext': 'mp4',
2035                 'title': 'Part 77   Sort a list of simple types in c#',
2036                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2037                 'upload_date': '20130831',
2038                 'uploader_id': 'kudvenkat',
2039                 'uploader': 'kudvenkat',
2040                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2041                 'like_count': int,
2042                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2043                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2044                 'live_status': 'not_live',
2045                 'categories': ['Education'],
2046                 'availability': 'public',
2047                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2048                 'tags': 'count:12',
2049                 'playable_in_embed': True,
2050                 'age_limit': 0,
2051                 'view_count': int,
2052                 'duration': 522,
2053                 'channel': 'kudvenkat',
2054                 'comment_count': int,
2055                 'channel_follower_count': int,
2056                 'chapters': list,
2057             },
2058             'params': {
2059                 'skip_download': True,
2060             },
2061         },
2062         {
2063             # another example of '};' in ytInitialData
2064             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2065             'only_matching': True,
2066         },
2067         {
2068             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2069             'only_matching': True,
2070         },
2071         {
2072             # https://github.com/ytdl-org/youtube-dl/pull/28094
2073             'url': 'OtqTfy26tG0',
2074             'info_dict': {
2075                 'id': 'OtqTfy26tG0',
2076                 'ext': 'mp4',
2077                 'title': 'Burn Out',
2078                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2079                 'upload_date': '20141120',
2080                 'uploader': 'The Cinematic Orchestra - Topic',
2081                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2082                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2083                 'artist': 'The Cinematic Orchestra',
2084                 'track': 'Burn Out',
2085                 'album': 'Every Day',
2086                 'like_count': int,
2087                 'live_status': 'not_live',
2088                 'alt_title': 'Burn Out',
2089                 'duration': 614,
2090                 'age_limit': 0,
2091                 'view_count': int,
2092                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2093                 'creator': 'The Cinematic Orchestra',
2094                 'channel': 'The Cinematic Orchestra',
2095                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2096                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2097                 'availability': 'public',
2098                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2099                 'categories': ['Music'],
2100                 'playable_in_embed': True,
2101                 'channel_follower_count': int
2102             },
2103             'params': {
2104                 'skip_download': True,
2105             },
2106         },
2107         {
2108             # controversial video, only works with bpctr when authenticated with cookies
2109             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2110             'only_matching': True,
2111         },
2112         {
2113             # controversial video, requires bpctr/contentCheckOk
2114             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2115             'info_dict': {
2116                 'id': 'SZJvDhaSDnc',
2117                 'ext': 'mp4',
2118                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2119                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2120                 'uploader': 'CBS Mornings',
2121                 'uploader_id': 'CBSThisMorning',
2122                 'upload_date': '20140716',
2123                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2124                 'duration': 170,
2125                 'categories': ['News & Politics'],
2126                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2127                 'view_count': int,
2128                 'channel': 'CBS Mornings',
2129                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2130                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2131                 'age_limit': 18,
2132                 'availability': 'needs_auth',
2133                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2134                 'like_count': int,
2135                 'live_status': 'not_live',
2136                 'playable_in_embed': True,
2137                 'channel_follower_count': int
2138             }
2139         },
2140         {
2141             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2142             'url': 'cBvYw8_A0vQ',
2143             'info_dict': {
2144                 'id': 'cBvYw8_A0vQ',
2145                 'ext': 'mp4',
2146                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2147                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2148                 'upload_date': '20201120',
2149                 'uploader': 'Walk around Japan',
2150                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2151                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2152                 'duration': 1456,
2153                 'categories': ['Travel & Events'],
2154                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2155                 'view_count': int,
2156                 'channel': 'Walk around Japan',
2157                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2158                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2159                 'age_limit': 0,
2160                 'availability': 'public',
2161                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2162                 'live_status': 'not_live',
2163                 'playable_in_embed': True,
2164                 'channel_follower_count': int
2165             },
2166             'params': {
2167                 'skip_download': True,
2168             },
2169         }, {
2170             # Has multiple audio streams
2171             'url': 'WaOKSUlf4TM',
2172             'only_matching': True
2173         }, {
2174             # Requires Premium: has format 141 when requested using YTM url
2175             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2176             'only_matching': True
2177         }, {
2178             # multiple subtitles with same lang_code
2179             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2180             'only_matching': True,
2181         }, {
2182             # Force use android client fallback
2183             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2184             'info_dict': {
2185                 'id': 'YOelRv7fMxY',
2186                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2187                 'ext': '3gp',
2188                 'upload_date': '20210624',
2189                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2190                 'uploader': 'colinfurze',
2191                 'uploader_id': 'colinfurze',
2192                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2193                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2194                 'duration': 596,
2195                 'categories': ['Entertainment'],
2196                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2197                 'view_count': int,
2198                 'channel': 'colinfurze',
2199                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2200                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2201                 'age_limit': 0,
2202                 'availability': 'public',
2203                 'like_count': int,
2204                 'live_status': 'not_live',
2205                 'playable_in_embed': True,
2206                 'channel_follower_count': int,
2207                 'chapters': list,
2208             },
2209             'params': {
2210                 'format': '17',  # 3gp format available on android
2211                 'extractor_args': {'youtube': {'player_client': ['android']}},
2212             },
2213         },
2214         {
2215             # Skip download of additional client configs (remix client config in this case)
2216             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2217             'only_matching': True,
2218             'params': {
2219                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2220             },
2221         }, {
2222             # shorts
2223             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2224             'only_matching': True,
2225         }, {
2226             'note': 'Storyboards',
2227             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2228             'info_dict': {
2229                 'id': '5KLPxDtMqe8',
2230                 'ext': 'mhtml',
2231                 'format_id': 'sb0',
2232                 'title': 'Your Brain is Plastic',
2233                 'uploader_id': 'scishow',
2234                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2235                 'upload_date': '20140324',
2236                 'uploader': 'SciShow',
2237                 'like_count': int,
2238                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2239                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2240                 'view_count': int,
2241                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2242                 'playable_in_embed': True,
2243                 'tags': 'count:12',
2244                 'uploader_url': 'http://www.youtube.com/user/scishow',
2245                 'availability': 'public',
2246                 'channel': 'SciShow',
2247                 'live_status': 'not_live',
2248                 'duration': 248,
2249                 'categories': ['Education'],
2250                 'age_limit': 0,
2251                 'channel_follower_count': int,
2252                 'chapters': list,
2253             }, 'params': {'format': 'mhtml', 'skip_download': True}
2254         }, {
2255             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2256             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2257             'info_dict': {
2258                 'id': '2NUZ8W2llS4',
2259                 'ext': 'mp4',
2260                 'title': 'The NP that test your phone performance 🙂',
2261                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2262                 'uploader': 'Leon Nguyen',
2263                 'uploader_id': 'VNSXIII',
2264                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2265                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2266                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2267                 'duration': 21,
2268                 'view_count': int,
2269                 'age_limit': 0,
2270                 'categories': ['Gaming'],
2271                 'tags': 'count:23',
2272                 'playable_in_embed': True,
2273                 'live_status': 'not_live',
2274                 'upload_date': '20220103',
2275                 'like_count': int,
2276                 'availability': 'public',
2277                 'channel': 'Leon Nguyen',
2278                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2279                 'comment_count': int,
2280                 'channel_follower_count': int
2281             }
2282         }, {
2283             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2284             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2285             'info_dict': {
2286                 'id': '2NUZ8W2llS4',
2287                 'ext': 'mp4',
2288                 'title': 'The NP that test your phone performance 🙂',
2289                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2290                 'uploader': 'Leon Nguyen',
2291                 'uploader_id': 'VNSXIII',
2292                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2293                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2294                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2295                 'duration': 21,
2296                 'view_count': int,
2297                 'age_limit': 0,
2298                 'categories': ['Gaming'],
2299                 'tags': 'count:23',
2300                 'playable_in_embed': True,
2301                 'live_status': 'not_live',
2302                 'upload_date': '20220102',
2303                 'like_count': int,
2304                 'availability': 'public',
2305                 'channel': 'Leon Nguyen',
2306                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2307                 'comment_count': int,
2308                 'channel_follower_count': int
2309             },
2310             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2311         }, {
2312             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2313             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2314             'info_dict': {
2315                 'id': 'mzZzzBU6lrM',
2316                 'ext': 'mp4',
2317                 'title': 'I Met GeorgeNotFound In Real Life...',
2318                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2319                 'uploader': 'Quackity',
2320                 'uploader_id': 'QuackityHQ',
2321                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2322                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2323                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2324                 'duration': 955,
2325                 'view_count': int,
2326                 'age_limit': 0,
2327                 'categories': ['Entertainment'],
2328                 'tags': 'count:26',
2329                 'playable_in_embed': True,
2330                 'live_status': 'not_live',
2331                 'release_timestamp': 1641172509,
2332                 'release_date': '20220103',
2333                 'upload_date': '20220103',
2334                 'like_count': int,
2335                 'availability': 'public',
2336                 'channel': 'Quackity',
2337                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2338                 'channel_follower_count': int
2339             }
2340         },
2341         {   # continuous livestream. Microformat upload date should be preferred.
2342             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2343             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2344             'info_dict': {
2345                 'id': 'kgx4WGK0oNU',
2346                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2347                 'ext': 'mp4',
2348                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2349                 'availability': 'public',
2350                 'age_limit': 0,
2351                 'release_timestamp': 1637975704,
2352                 'upload_date': '20210619',
2353                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2354                 'live_status': 'is_live',
2355                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2356                 'uploader': '阿鲍Abao',
2357                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2358                 'channel': 'Abao in Tokyo',
2359                 'channel_follower_count': int,
2360                 'release_date': '20211127',
2361                 'tags': 'count:39',
2362                 'categories': ['People & Blogs'],
2363                 'like_count': int,
2364                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2365                 'view_count': int,
2366                 'playable_in_embed': True,
2367                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2368                 'concurrent_view_count': int,
2369             },
2370             'params': {'skip_download': True}
2371         }, {
2372             # Story. Requires specific player params to work.
2373             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2374             'info_dict': {
2375                 'id': 'vv8qTUWmulI',
2376                 'ext': 'mp4',
2377                 'availability': 'unlisted',
2378                 'view_count': int,
2379                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2380                 'upload_date': '20220526',
2381                 'categories': ['Education'],
2382                 'title': 'Story',
2383                 'channel': 'IT\'S HISTORY',
2384                 'description': '',
2385                 'uploader_id': 'BlastfromthePast',
2386                 'duration': 12,
2387                 'uploader': 'IT\'S HISTORY',
2388                 'playable_in_embed': True,
2389                 'age_limit': 0,
2390                 'live_status': 'not_live',
2391                 'tags': [],
2392                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2393                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2394                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2395             },
2396             'skip': 'stories get removed after some period of time',
2397         }, {
2398             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2399             'info_dict': {
2400                 'id': 'tjjjtzRLHvA',
2401                 'ext': 'mp4',
2402                 'title': 'ハッシュタグ無し };if window.ytcsi',
2403                 'upload_date': '20220323',
2404                 'like_count': int,
2405                 'availability': 'unlisted',
2406                 'channel': 'nao20010128nao',
2407                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2408                 'age_limit': 0,
2409                 'uploader': 'nao20010128nao',
2410                 'uploader_id': 'nao20010128nao',
2411                 'categories': ['Music'],
2412                 'view_count': int,
2413                 'description': '',
2414                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2415                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2416                 'live_status': 'not_live',
2417                 'playable_in_embed': True,
2418                 'channel_follower_count': int,
2419                 'duration': 6,
2420                 'tags': [],
2421                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2422             }
2423         }, {
2424             # Prefer primary title+description language metadata by default
2425             # Do not prefer translated description if primary is empty
2426             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2427             'info_dict': {
2428                 'id': 'el3E4MbxRqQ',
2429                 'ext': 'mp4',
2430                 'title': 'dlp test video 2 - primary sv no desc',
2431                 'description': '',
2432                 'channel': 'cole-dlp-test-acc',
2433                 'tags': [],
2434                 'view_count': int,
2435                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2436                 'like_count': int,
2437                 'playable_in_embed': True,
2438                 'availability': 'unlisted',
2439                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2440                 'age_limit': 0,
2441                 'duration': 5,
2442                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2443                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2444                 'live_status': 'not_live',
2445                 'upload_date': '20220908',
2446                 'categories': ['People & Blogs'],
2447                 'uploader': 'cole-dlp-test-acc',
2448                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2449             },
2450             'params': {'skip_download': True}
2451         }, {
2452             # Extractor argument: prefer translated title+description
2453             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2454             'info_dict': {
2455                 'id': 'gHKT4uU8Zng',
2456                 'ext': 'mp4',
2457                 'channel': 'cole-dlp-test-acc',
2458                 'tags': [],
2459                 'duration': 5,
2460                 'live_status': 'not_live',
2461                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2462                 'upload_date': '20220728',
2463                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2464                 'view_count': int,
2465                 'categories': ['People & Blogs'],
2466                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2467                 'title': 'dlp test video title translated (fr)',
2468                 'availability': 'public',
2469                 'uploader': 'cole-dlp-test-acc',
2470                 'age_limit': 0,
2471                 'description': 'dlp test video description translated (fr)',
2472                 'playable_in_embed': True,
2473                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2474                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2475             },
2476             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2477             'expected_warnings': [r'Preferring "fr" translated fields'],
2478         }, {
2479             'note': '6 channel audio',
2480             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2481             'only_matching': True,
2482         }
2483     ]
2484
2485     _WEBPAGE_TESTS = [
2486         # YouTube <object> embed
2487         {
2488             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2489             'md5': '873c81d308b979f0e23ee7e620b312a3',
2490             'info_dict': {
2491                 'id': 'msN87y-iEx0',
2492                 'ext': 'mp4',
2493                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2494                 'upload_date': '20080526',
2495                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2496                 'uploader': 'Christopher Sykes',
2497                 'uploader_id': 'ChristopherJSykes',
2498                 'age_limit': 0,
2499                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2500                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2501                 'playable_in_embed': True,
2502                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2503                 'like_count': int,
2504                 'comment_count': int,
2505                 'channel': 'Christopher Sykes',
2506                 'live_status': 'not_live',
2507                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2508                 'availability': 'public',
2509                 'duration': 195,
2510                 'view_count': int,
2511                 'categories': ['Science & Technology'],
2512                 'channel_follower_count': int,
2513                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2514             },
2515             'params': {
2516                 'skip_download': True,
2517             }
2518         },
2519     ]
2520
2521     @classmethod
2522     def suitable(cls, url):
2523         from ..utils import parse_qs
2524
2525         qs = parse_qs(url)
2526         if qs.get('list', [None])[0]:
2527             return False
2528         return super().suitable(url)
2529
2530     def __init__(self, *args, **kwargs):
2531         super().__init__(*args, **kwargs)
2532         self._code_cache = {}
2533         self._player_cache = {}
2534
2535     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2536         lock = threading.Lock()
2537         start_time = time.time()
2538         formats = [f for f in formats if f.get('is_from_start')]
2539
2540         def refetch_manifest(format_id, delay):
2541             nonlocal formats, start_time, is_live
2542             if time.time() <= start_time + delay:
2543                 return
2544
2545             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2546             video_details = traverse_obj(
2547                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2548             microformats = traverse_obj(
2549                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2550                 expected_type=dict, default=[])
2551             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2552             is_live = live_status == 'is_live'
2553             start_time = time.time()
2554
2555         def mpd_feed(format_id, delay):
2556             """
2557             @returns (manifest_url, manifest_stream_number, is_live) or None
2558             """
2559             with lock:
2560                 refetch_manifest(format_id, delay)
2561
2562             f = next((f for f in formats if f['format_id'] == format_id), None)
2563             if not f:
2564                 if not is_live:
2565                     self.to_screen(f'{video_id}: Video is no longer live')
2566                 else:
2567                     self.report_warning(
2568                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2569                 return None
2570             return f['manifest_url'], f['manifest_stream_number'], is_live
2571
2572         for f in formats:
2573             f['is_live'] = is_live
2574             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2575                                     live_start_time, mpd_feed, not is_live and f.copy())
2576             if is_live:
2577                 f['fragments'] = gen
2578                 f['protocol'] = 'http_dash_segments_generator'
2579             else:
2580                 f['fragments'] = LazyList(gen({}))
2581                 del f['is_from_start']
2582
2583     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2584         FETCH_SPAN, MAX_DURATION = 5, 432000
2585
2586         mpd_url, stream_number, is_live = None, None, True
2587
2588         begin_index = 0
2589         download_start_time = ctx.get('start') or time.time()
2590
2591         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2592         if lack_early_segments:
2593             self.report_warning(bug_reports_message(
2594                 'Starting download from the last 120 hours of the live stream since '
2595                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2596             lack_early_segments = True
2597
2598         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2599         fragments, fragment_base_url = None, None
2600
2601         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2602             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2603             # Obtain from MPD's maximum seq value
2604             old_mpd_url = mpd_url
2605             last_error = ctx.pop('last_error', None)
2606             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2607             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2608                                                or (mpd_url, stream_number, False))
2609             if not refresh_sequence:
2610                 if expire_fast and not is_live:
2611                     return False, last_seq
2612                 elif old_mpd_url == mpd_url:
2613                     return True, last_seq
2614             if manifestless_orig_fmt:
2615                 fmt_info = manifestless_orig_fmt
2616             else:
2617                 try:
2618                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2619                         mpd_url, None, note=False, errnote=False, fatal=False)
2620                 except ExtractorError:
2621                     fmts = None
2622                 if not fmts:
2623                     no_fragment_score += 2
2624                     return False, last_seq
2625                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2626             fragments = fmt_info['fragments']
2627             fragment_base_url = fmt_info['fragment_base_url']
2628             assert fragment_base_url
2629
2630             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2631             return True, _last_seq
2632
2633         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2634         while is_live:
2635             fetch_time = time.time()
2636             if no_fragment_score > 30:
2637                 return
2638             if last_segment_url:
2639                 # Obtain from "X-Head-Seqnum" header value from each segment
2640                 try:
2641                     urlh = self._request_webpage(
2642                         last_segment_url, None, note=False, errnote=False, fatal=False)
2643                 except ExtractorError:
2644                     urlh = None
2645                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2646                 if last_seq is None:
2647                     no_fragment_score += 2
2648                     last_segment_url = None
2649                     continue
2650             else:
2651                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2652                 no_fragment_score += 2
2653                 if not should_continue:
2654                     continue
2655
2656             if known_idx > last_seq:
2657                 last_segment_url = None
2658                 continue
2659
2660             last_seq += 1
2661
2662             if begin_index < 0 and known_idx < 0:
2663                 # skip from the start when it's negative value
2664                 known_idx = last_seq + begin_index
2665             if lack_early_segments:
2666                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2667             try:
2668                 for idx in range(known_idx, last_seq):
2669                     # do not update sequence here or you'll get skipped some part of it
2670                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2671                     if not should_continue:
2672                         known_idx = idx - 1
2673                         raise ExtractorError('breaking out of outer loop')
2674                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2675                     yield {
2676                         'url': last_segment_url,
2677                         'fragment_count': last_seq,
2678                     }
2679                 if known_idx == last_seq:
2680                     no_fragment_score += 5
2681                 else:
2682                     no_fragment_score = 0
2683                 known_idx = last_seq
2684             except ExtractorError:
2685                 continue
2686
2687             if manifestless_orig_fmt:
2688                 # Stop at the first iteration if running for post-live manifestless;
2689                 # fragment count no longer increase since it starts
2690                 break
2691
2692             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2693
2694     def _extract_player_url(self, *ytcfgs, webpage=None):
2695         player_url = traverse_obj(
2696             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2697             get_all=False, expected_type=str)
2698         if not player_url:
2699             return
2700         return urljoin('https://www.youtube.com', player_url)
2701
2702     def _download_player_url(self, video_id, fatal=False):
2703         res = self._download_webpage(
2704             'https://www.youtube.com/iframe_api',
2705             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2706         if res:
2707             player_version = self._search_regex(
2708                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2709             if player_version:
2710                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2711
2712     def _signature_cache_id(self, example_sig):
2713         """ Return a string representation of a signature """
2714         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2715
2716     @classmethod
2717     def _extract_player_info(cls, player_url):
2718         for player_re in cls._PLAYER_INFO_RE:
2719             id_m = re.search(player_re, player_url)
2720             if id_m:
2721                 break
2722         else:
2723             raise ExtractorError('Cannot identify player %r' % player_url)
2724         return id_m.group('id')
2725
2726     def _load_player(self, video_id, player_url, fatal=True):
2727         player_id = self._extract_player_info(player_url)
2728         if player_id not in self._code_cache:
2729             code = self._download_webpage(
2730                 player_url, video_id, fatal=fatal,
2731                 note='Downloading player ' + player_id,
2732                 errnote='Download of %s failed' % player_url)
2733             if code:
2734                 self._code_cache[player_id] = code
2735         return self._code_cache.get(player_id)
2736
2737     def _extract_signature_function(self, video_id, player_url, example_sig):
2738         player_id = self._extract_player_info(player_url)
2739
2740         # Read from filesystem cache
2741         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2742         assert os.path.basename(func_id) == func_id
2743
2744         self.write_debug(f'Extracting signature function {func_id}')
2745         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2746
2747         if not cache_spec:
2748             code = self._load_player(video_id, player_url)
2749         if code:
2750             res = self._parse_sig_js(code)
2751             test_string = ''.join(map(chr, range(len(example_sig))))
2752             cache_spec = [ord(c) for c in res(test_string)]
2753             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2754
2755         return lambda s: ''.join(s[i] for i in cache_spec)
2756
2757     def _print_sig_code(self, func, example_sig):
2758         if not self.get_param('youtube_print_sig_code'):
2759             return
2760
2761         def gen_sig_code(idxs):
2762             def _genslice(start, end, step):
2763                 starts = '' if start == 0 else str(start)
2764                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2765                 steps = '' if step == 1 else (':%d' % step)
2766                 return f's[{starts}{ends}{steps}]'
2767
2768             step = None
2769             # Quelch pyflakes warnings - start will be set when step is set
2770             start = '(Never used)'
2771             for i, prev in zip(idxs[1:], idxs[:-1]):
2772                 if step is not None:
2773                     if i - prev == step:
2774                         continue
2775                     yield _genslice(start, prev, step)
2776                     step = None
2777                     continue
2778                 if i - prev in [-1, 1]:
2779                     step = i - prev
2780                     start = prev
2781                     continue
2782                 else:
2783                     yield 's[%d]' % prev
2784             if step is None:
2785                 yield 's[%d]' % i
2786             else:
2787                 yield _genslice(start, i, step)
2788
2789         test_string = ''.join(map(chr, range(len(example_sig))))
2790         cache_res = func(test_string)
2791         cache_spec = [ord(c) for c in cache_res]
2792         expr_code = ' + '.join(gen_sig_code(cache_spec))
2793         signature_id_tuple = '(%s)' % (
2794             ', '.join(str(len(p)) for p in example_sig.split('.')))
2795         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2796                 '    return %s\n') % (signature_id_tuple, expr_code)
2797         self.to_screen('Extracted signature function:\n' + code)
2798
2799     def _parse_sig_js(self, jscode):
2800         funcname = self._search_regex(
2801             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2802              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2803              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2804              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2805              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2806              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2807              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2808              # Obsolete patterns
2809              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2810              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2811              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2812              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2813              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2814              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2815              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2816              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2817             jscode, 'Initial JS player signature function name', group='sig')
2818
2819         jsi = JSInterpreter(jscode)
2820         initial_function = jsi.extract_function(funcname)
2821         return lambda s: initial_function([s])
2822
2823     def _cached(self, func, *cache_id):
2824         def inner(*args, **kwargs):
2825             if cache_id not in self._player_cache:
2826                 try:
2827                     self._player_cache[cache_id] = func(*args, **kwargs)
2828                 except ExtractorError as e:
2829                     self._player_cache[cache_id] = e
2830                 except Exception as e:
2831                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
2832
2833             ret = self._player_cache[cache_id]
2834             if isinstance(ret, Exception):
2835                 raise ret
2836             return ret
2837         return inner
2838
2839     def _decrypt_signature(self, s, video_id, player_url):
2840         """Turn the encrypted s field into a working signature"""
2841         extract_sig = self._cached(
2842             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
2843         func = extract_sig(video_id, player_url, s)
2844         self._print_sig_code(func, s)
2845         return func(s)
2846
2847     def _decrypt_nsig(self, s, video_id, player_url):
2848         """Turn the encrypted n field into a working signature"""
2849         if player_url is None:
2850             raise ExtractorError('Cannot decrypt nsig without player_url')
2851         player_url = urljoin('https://www.youtube.com', player_url)
2852
2853         try:
2854             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
2855         except ExtractorError as e:
2856             raise ExtractorError('Unable to extract nsig function code', cause=e)
2857         if self.get_param('youtube_print_sig_code'):
2858             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2859
2860         try:
2861             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
2862             ret = extract_nsig(jsi, func_code)(s)
2863         except JSInterpreter.Exception as e:
2864             try:
2865                 jsi = PhantomJSwrapper(self, timeout=5000)
2866             except ExtractorError:
2867                 raise e
2868             self.report_warning(
2869                 f'Native nsig extraction failed: Trying with PhantomJS\n'
2870                 f'         n = {s} ; player = {player_url}', video_id)
2871             self.write_debug(e, only_once=True)
2872
2873             args, func_body = func_code
2874             ret = jsi.execute(
2875                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
2876                 video_id=video_id, note='Executing signature code').strip()
2877
2878         self.write_debug(f'Decrypted nsig {s} => {ret}')
2879         return ret
2880
2881     def _extract_n_function_name(self, jscode):
2882         funcname, idx = self._search_regex(
2883             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2884             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2885         if not idx:
2886             return funcname
2887
2888         return json.loads(js_to_json(self._search_regex(
2889             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
2890             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
2891
2892     def _extract_n_function_code(self, video_id, player_url):
2893         player_id = self._extract_player_info(player_url)
2894         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
2895         jscode = func_code or self._load_player(video_id, player_url)
2896         jsi = JSInterpreter(jscode)
2897
2898         if func_code:
2899             return jsi, player_id, func_code
2900
2901         func_name = self._extract_n_function_name(jscode)
2902
2903         # For redundancy
2904         func_code = self._search_regex(
2905             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
2906                      # NB: The end of the regex is intentionally kept strict
2907                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
2908             jscode, 'nsig function', group=('var', 'code'), default=None)
2909         if func_code:
2910             func_code = ([func_code[0]], func_code[1])
2911         else:
2912             self.write_debug('Extracting nsig function with jsinterp')
2913             func_code = jsi.extract_function_code(func_name)
2914
2915         self.cache.store('youtube-nsig', player_id, func_code)
2916         return jsi, player_id, func_code
2917
2918     def _extract_n_function_from_code(self, jsi, func_code):
2919         func = jsi.extract_function_from_code(*func_code)
2920
2921         def extract_nsig(s):
2922             try:
2923                 ret = func([s])
2924             except JSInterpreter.Exception:
2925                 raise
2926             except Exception as e:
2927                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
2928
2929             if ret.startswith('enhanced_except_'):
2930                 raise JSInterpreter.Exception('Signature function returned an exception')
2931             return ret
2932
2933         return extract_nsig
2934
2935     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2936         """
2937         Extract signatureTimestamp (sts)
2938         Required to tell API what sig/player version is in use.
2939         """
2940         sts = None
2941         if isinstance(ytcfg, dict):
2942             sts = int_or_none(ytcfg.get('STS'))
2943
2944         if not sts:
2945             # Attempt to extract from player
2946             if player_url is None:
2947                 error_msg = 'Cannot extract signature timestamp without player_url.'
2948                 if fatal:
2949                     raise ExtractorError(error_msg)
2950                 self.report_warning(error_msg)
2951                 return
2952             code = self._load_player(video_id, player_url, fatal=fatal)
2953             if code:
2954                 sts = int_or_none(self._search_regex(
2955                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2956                     'JS player signature timestamp', group='sts', fatal=fatal))
2957         return sts
2958
2959     def _mark_watched(self, video_id, player_responses):
2960         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2961             label = 'fully ' if is_full else ''
2962             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2963                             expected_type=url_or_none)
2964             if not url:
2965                 self.report_warning(f'Unable to mark {label}watched')
2966                 return
2967             parsed_url = urllib.parse.urlparse(url)
2968             qs = urllib.parse.parse_qs(parsed_url.query)
2969
2970             # cpn generation algorithm is reverse engineered from base.js.
2971             # In fact it works even with dummy cpn.
2972             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2973             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2974
2975             # # more consistent results setting it to right before the end
2976             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2977
2978             qs.update({
2979                 'ver': ['2'],
2980                 'cpn': [cpn],
2981                 'cmt': video_length,
2982                 'el': 'detailpage',  # otherwise defaults to "shorts"
2983             })
2984
2985             if is_full:
2986                 # these seem to mark watchtime "history" in the real world
2987                 # they're required, so send in a single value
2988                 qs.update({
2989                     'st': 0,
2990                     'et': video_length,
2991                 })
2992
2993             url = urllib.parse.urlunparse(
2994                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2995
2996             self._download_webpage(
2997                 url, video_id, f'Marking {label}watched',
2998                 'Unable to mark watched', fatal=False)
2999
3000     @classmethod
3001     def _extract_from_webpage(cls, url, webpage):
3002         # Invidious Instances
3003         # https://github.com/yt-dlp/yt-dlp/issues/195
3004         # https://github.com/iv-org/invidious/pull/1730
3005         mobj = re.search(
3006             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
3007             webpage)
3008         if mobj:
3009             yield cls.url_result(mobj.group('url'), cls)
3010             raise cls.StopExtraction()
3011
3012         yield from super()._extract_from_webpage(url, webpage)
3013
3014         # lazyYT YouTube embed
3015         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
3016             yield cls.url_result(unescapeHTML(id_), cls, id_)
3017
3018         # Wordpress "YouTube Video Importer" plugin
3019         for m in re.findall(r'''(?x)<div[^>]+
3020                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
3021                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
3022             yield cls.url_result(m[-1], cls, m[-1])
3023
3024     @classmethod
3025     def extract_id(cls, url):
3026         video_id = cls.get_temp_id(url)
3027         if not video_id:
3028             raise ExtractorError(f'Invalid URL: {url}')
3029         return video_id
3030
3031     def _extract_chapters_from_json(self, data, duration):
3032         chapter_list = traverse_obj(
3033             data, (
3034                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3035                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3036             ), expected_type=list)
3037
3038         return self._extract_chapters(
3039             chapter_list,
3040             chapter_time=lambda chapter: float_or_none(
3041                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3042             chapter_title=lambda chapter: traverse_obj(
3043                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3044             duration=duration)
3045
3046     def _extract_chapters_from_engagement_panel(self, data, duration):
3047         content_list = traverse_obj(
3048             data,
3049             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3050             expected_type=list, default=[])
3051         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3052         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3053
3054         return next(filter(None, (
3055             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3056                                    chapter_time, chapter_title, duration)
3057             for contents in content_list)), [])
3058
3059     def _extract_chapters_from_description(self, description, duration):
3060         duration_re = r'(?:\d+:)?\d{1,2}:\d{2}'
3061         sep_re = r'(?m)^\s*(%s)\b\W*\s(%s)\s*$'
3062         return self._extract_chapters(
3063             re.findall(sep_re % (duration_re, r'.+?'), description or ''),
3064             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
3065             duration=duration, strict=False) or self._extract_chapters(
3066             re.findall(sep_re % (r'.+?', duration_re), description or ''),
3067             chapter_time=lambda x: parse_duration(x[1]), chapter_title=lambda x: x[0],
3068             duration=duration, strict=False)
3069
3070     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
3071         if not duration:
3072             return
3073         chapter_list = [{
3074             'start_time': chapter_time(chapter),
3075             'title': chapter_title(chapter),
3076         } for chapter in chapter_list or []]
3077         if not strict:
3078             chapter_list.sort(key=lambda c: c['start_time'] or 0)
3079
3080         chapters = [{'start_time': 0}]
3081         for idx, chapter in enumerate(chapter_list):
3082             if chapter['start_time'] is None:
3083                 self.report_warning(f'Incomplete chapter {idx}')
3084             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
3085                 chapters.append(chapter)
3086             elif chapter not in chapters:
3087                 self.report_warning(
3088                     f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
3089         return chapters[1:]
3090
3091     def _extract_comment(self, comment_renderer, parent=None):
3092         comment_id = comment_renderer.get('commentId')
3093         if not comment_id:
3094             return
3095
3096         text = self._get_text(comment_renderer, 'contentText')
3097
3098         # Timestamp is an estimate calculated from the current time and time_text
3099         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3100         timestamp = self._parse_time_text(time_text)
3101
3102         author = self._get_text(comment_renderer, 'authorText')
3103         author_id = try_get(comment_renderer,
3104                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3105
3106         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3107                                                        lambda x: x['likeCount']), str)) or 0
3108         author_thumbnail = try_get(comment_renderer,
3109                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3110
3111         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3112         is_favorited = 'creatorHeart' in (try_get(
3113             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3114         return {
3115             'id': comment_id,
3116             'text': text,
3117             'timestamp': timestamp,
3118             'time_text': time_text,
3119             'like_count': votes,
3120             'is_favorited': is_favorited,
3121             'author': author,
3122             'author_id': author_id,
3123             'author_thumbnail': author_thumbnail,
3124             'author_is_uploader': author_is_uploader,
3125             'parent': parent or 'root'
3126         }
3127
3128     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3129
3130         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3131
3132         def extract_header(contents):
3133             _continuation = None
3134             for content in contents:
3135                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3136                 expected_comment_count = self._get_count(
3137                     comments_header_renderer, 'countText', 'commentsCount')
3138
3139                 if expected_comment_count:
3140                     tracker['est_total'] = expected_comment_count
3141                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3142                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3143
3144                 sort_menu_item = try_get(
3145                     comments_header_renderer,
3146                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3147                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3148
3149                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3150                 if not _continuation:
3151                     continue
3152
3153                 sort_text = str_or_none(sort_menu_item.get('title'))
3154                 if not sort_text:
3155                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3156                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3157                 break
3158             return _continuation
3159
3160         def extract_thread(contents):
3161             if not parent:
3162                 tracker['current_page_thread'] = 0
3163             for content in contents:
3164                 if not parent and tracker['total_parent_comments'] >= max_parents:
3165                     yield
3166                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3167                 comment_renderer = get_first(
3168                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3169                     expected_type=dict, default={})
3170
3171                 comment = self._extract_comment(comment_renderer, parent)
3172                 if not comment:
3173                     continue
3174
3175                 tracker['running_total'] += 1
3176                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3177                 yield comment
3178
3179                 # Attempt to get the replies
3180                 comment_replies_renderer = try_get(
3181                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3182
3183                 if comment_replies_renderer:
3184                     tracker['current_page_thread'] += 1
3185                     comment_entries_iter = self._comment_entries(
3186                         comment_replies_renderer, ytcfg, video_id,
3187                         parent=comment.get('id'), tracker=tracker)
3188                     yield from itertools.islice(comment_entries_iter, min(
3189                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3190
3191         # Keeps track of counts across recursive calls
3192         if not tracker:
3193             tracker = dict(
3194                 running_total=0,
3195                 est_total=0,
3196                 current_page_thread=0,
3197                 total_parent_comments=0,
3198                 total_reply_comments=0)
3199
3200         # TODO: Deprecated
3201         # YouTube comments have a max depth of 2
3202         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3203         if max_depth:
3204             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3205                                                 'Set max replies in the max-comments extractor argument instead')
3206         if max_depth == 1 and parent:
3207             return
3208
3209         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3210             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3211
3212         continuation = self._extract_continuation(root_continuation_data)
3213
3214         response = None
3215         is_forced_continuation = False
3216         is_first_continuation = parent is None
3217         if is_first_continuation and not continuation:
3218             # Sometimes you can get comments by generating the continuation yourself,
3219             # even if YouTube initially reports them being disabled - e.g. stories comments.
3220             # Note: if the comment section is actually disabled, YouTube may return a response with
3221             # required check_get_keys missing. So we will disable that check initially in this case.
3222             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3223             is_forced_continuation = True
3224
3225         for page_num in itertools.count(0):
3226             if not continuation:
3227                 break
3228             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3229             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3230             if page_num == 0:
3231                 if is_first_continuation:
3232                     note_prefix = 'Downloading comment section API JSON'
3233                 else:
3234                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3235                         tracker['current_page_thread'], comment_prog_str)
3236             else:
3237                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3238                     '       ' if parent else '', ' replies' if parent else '',
3239                     page_num, comment_prog_str)
3240             try:
3241                 response = self._extract_response(
3242                     item_id=None, query=continuation,
3243                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3244                     check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3245             except ExtractorError as e:
3246                 # Ignore incomplete data error for replies if retries didn't work.
3247                 # This is to allow any other parent comments and comment threads to be downloaded.
3248                 # See: https://github.com/yt-dlp/yt-dlp/issues/4669
3249                 if 'incomplete data' in str(e).lower() and parent and self.get_param('ignoreerrors') is True:
3250                     self.report_warning(
3251                         'Received incomplete data for a comment reply thread and retrying did not help. '
3252                         'Ignoring to let other comments be downloaded.')
3253                 else:
3254                     raise
3255             is_forced_continuation = False
3256             continuation_contents = traverse_obj(
3257                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3258
3259             continuation = None
3260             for continuation_section in continuation_contents:
3261                 continuation_items = traverse_obj(
3262                     continuation_section,
3263                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3264                     get_all=False, expected_type=list) or []
3265                 if is_first_continuation:
3266                     continuation = extract_header(continuation_items)
3267                     is_first_continuation = False
3268                     if continuation:
3269                         break
3270                     continue
3271
3272                 for entry in extract_thread(continuation_items):
3273                     if not entry:
3274                         return
3275                     yield entry
3276                 continuation = self._extract_continuation({'contents': continuation_items})
3277                 if continuation:
3278                     break
3279
3280         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3281         if message and not parent and tracker['running_total'] == 0:
3282             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3283             raise self.CommentsDisabled
3284
3285     @staticmethod
3286     def _generate_comment_continuation(video_id):
3287         """
3288         Generates initial comment section continuation token from given video id
3289         """
3290         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3291         return base64.b64encode(token.encode()).decode()
3292
3293     def _get_comments(self, ytcfg, video_id, contents, webpage):
3294         """Entry for comment extraction"""
3295         def _real_comment_extract(contents):
3296             renderer = next((
3297                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3298                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3299             yield from self._comment_entries(renderer, ytcfg, video_id)
3300
3301         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3302         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3303
3304     @staticmethod
3305     def _get_checkok_params():
3306         return {'contentCheckOk': True, 'racyCheckOk': True}
3307
3308     @classmethod
3309     def _generate_player_context(cls, sts=None):
3310         context = {
3311             'html5Preference': 'HTML5_PREF_WANTS',
3312         }
3313         if sts is not None:
3314             context['signatureTimestamp'] = sts
3315         return {
3316             'playbackContext': {
3317                 'contentPlaybackContext': context
3318             },
3319             **cls._get_checkok_params()
3320         }
3321
3322     @staticmethod
3323     def _is_agegated(player_response):
3324         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3325             return True
3326
3327         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3328         AGE_GATE_REASONS = (
3329             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3330             'age_verification_required', 'age_check_required',  # status
3331         )
3332         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3333
3334     @staticmethod
3335     def _is_unplayable(player_response):
3336         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3337
3338     _STORY_PLAYER_PARAMS = '8AEB'
3339
3340     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3341
3342         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3343         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3344         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3345         headers = self.generate_api_headers(
3346             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3347
3348         yt_query = {
3349             'videoId': video_id,
3350         }
3351         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3352             yt_query['params'] = self._STORY_PLAYER_PARAMS
3353
3354         yt_query.update(self._generate_player_context(sts))
3355         return self._extract_response(
3356             item_id=video_id, ep='player', query=yt_query,
3357             ytcfg=player_ytcfg, headers=headers, fatal=True,
3358             default_client=client,
3359             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3360         ) or None
3361
3362     def _get_requested_clients(self, url, smuggled_data):
3363         requested_clients = []
3364         default = ['android', 'web']
3365         allowed_clients = sorted(
3366             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3367             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3368         for client in self._configuration_arg('player_client'):
3369             if client in allowed_clients:
3370                 requested_clients.append(client)
3371             elif client == 'default':
3372                 requested_clients.extend(default)
3373             elif client == 'all':
3374                 requested_clients.extend(allowed_clients)
3375             else:
3376                 self.report_warning(f'Skipping unsupported client {client}')
3377         if not requested_clients:
3378             requested_clients = default
3379
3380         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3381             requested_clients.extend(
3382                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3383
3384         return orderedSet(requested_clients)
3385
3386     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3387         initial_pr = None
3388         if webpage:
3389             initial_pr = self._search_json(
3390                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3391
3392         all_clients = set(clients)
3393         clients = clients[::-1]
3394         prs = []
3395
3396         def append_client(*client_names):
3397             """ Append the first client name that exists but not already used """
3398             for client_name in client_names:
3399                 actual_client = _split_innertube_client(client_name)[0]
3400                 if actual_client in INNERTUBE_CLIENTS:
3401                     if actual_client not in all_clients:
3402                         clients.append(client_name)
3403                         all_clients.add(actual_client)
3404                         return
3405
3406         # Android player_response does not have microFormats which are needed for
3407         # extraction of some data. So we return the initial_pr with formats
3408         # stripped out even if not requested by the user
3409         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3410         if initial_pr:
3411             pr = dict(initial_pr)
3412             pr['streamingData'] = None
3413             prs.append(pr)
3414
3415         last_error = None
3416         tried_iframe_fallback = False
3417         player_url = None
3418         while clients:
3419             client, base_client, variant = _split_innertube_client(clients.pop())
3420             player_ytcfg = master_ytcfg if client == 'web' else {}
3421             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3422                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3423
3424             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3425             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3426             if 'js' in self._configuration_arg('player_skip'):
3427                 require_js_player = False
3428                 player_url = None
3429
3430             if not player_url and not tried_iframe_fallback and require_js_player:
3431                 player_url = self._download_player_url(video_id)
3432                 tried_iframe_fallback = True
3433
3434             try:
3435                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3436                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3437             except ExtractorError as e:
3438                 if last_error:
3439                     self.report_warning(last_error)
3440                 last_error = e
3441                 continue
3442
3443             if pr:
3444                 # YouTube may return a different video player response than expected.
3445                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3446                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3447                 if pr_video_id and pr_video_id != video_id:
3448                     self.report_warning(
3449                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3450                 else:
3451                     prs.append(pr)
3452
3453             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3454             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3455                 append_client(f'{base_client}_creator')
3456             elif self._is_agegated(pr):
3457                 if variant == 'tv_embedded':
3458                     append_client(f'{base_client}_embedded')
3459                 elif not variant:
3460                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3461
3462         if last_error:
3463             if not len(prs):
3464                 raise last_error
3465             self.report_warning(last_error)
3466         return prs, player_url
3467
3468     def _needs_live_processing(self, live_status, duration):
3469         if (live_status == 'is_live' and self.get_param('live_from_start')
3470                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3471             return live_status
3472
3473     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3474         itags, stream_ids = {}, []
3475         itag_qualities, res_qualities = {}, {0: None}
3476         q = qualities([
3477             # Normally tiny is the smallest video-only formats. But
3478             # audio-only formats with unknown quality may get tagged as tiny
3479             'tiny',
3480             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3481             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3482         ])
3483         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3484
3485         for fmt in streaming_formats:
3486             if fmt.get('targetDurationSec'):
3487                 continue
3488
3489             itag = str_or_none(fmt.get('itag'))
3490             audio_track = fmt.get('audioTrack') or {}
3491             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3492             if stream_id in stream_ids:
3493                 continue
3494
3495             quality = fmt.get('quality')
3496             height = int_or_none(fmt.get('height'))
3497             if quality == 'tiny' or not quality:
3498                 quality = fmt.get('audioQuality', '').lower() or quality
3499             # The 3gp format (17) in android client has a quality of "small",
3500             # but is actually worse than other formats
3501             if itag == '17':
3502                 quality = 'tiny'
3503             if quality:
3504                 if itag:
3505                     itag_qualities[itag] = quality
3506                 if height:
3507                     res_qualities[height] = quality
3508             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3509             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3510             # number of fragment that would subsequently requested with (`&sq=N`)
3511             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3512                 continue
3513
3514             fmt_url = fmt.get('url')
3515             if not fmt_url:
3516                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3517                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3518                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3519                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3520                     continue
3521                 try:
3522                     fmt_url += '&%s=%s' % (
3523                         traverse_obj(sc, ('sp', -1)) or 'signature',
3524                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3525                     )
3526                 except ExtractorError as e:
3527                     self.report_warning('Signature extraction failed: Some formats may be missing',
3528                                         video_id=video_id, only_once=True)
3529                     self.write_debug(e, only_once=True)
3530                     continue
3531
3532             query = parse_qs(fmt_url)
3533             throttled = False
3534             if query.get('n'):
3535                 try:
3536                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3537                     fmt_url = update_url_query(fmt_url, {
3538                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3539                     })
3540                 except ExtractorError as e:
3541                     phantomjs_hint = ''
3542                     if isinstance(e, JSInterpreter.Exception):
3543                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3544                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3545                     if player_url:
3546                         self.report_warning(
3547                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3548                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3549                         self.write_debug(e, only_once=True)
3550                     else:
3551                         self.report_warning(
3552                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3553                             video_id=video_id, only_once=True)
3554                     throttled = True
3555
3556             if itag:
3557                 itags[itag] = 'https'
3558                 stream_ids.append(stream_id)
3559
3560             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3561             language_preference = (
3562                 10 if audio_track.get('audioIsDefault') and 10
3563                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3564                 else -1)
3565             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3566             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3567             # Make sure to avoid false positives with small duration differences.
3568             # E.g. __2ABJjxzNo, ySuUZEjARPY
3569             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3570             if is_damaged:
3571                 self.report_warning(
3572                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3573             dct = {
3574                 'asr': int_or_none(fmt.get('audioSampleRate')),
3575                 'filesize': int_or_none(fmt.get('contentLength')),
3576                 'format_id': itag,
3577                 'format_note': join_nonempty(
3578                     '%s%s' % (audio_track.get('displayName') or '',
3579                               ' (default)' if language_preference > 0 else ''),
3580                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3581                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3582                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3583                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3584                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3585                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3586                 'fps': int_or_none(fmt.get('fps')) or None,
3587                 'audio_channels': fmt.get('audioChannels'),
3588                 'height': height,
3589                 'quality': q(quality),
3590                 'has_drm': bool(fmt.get('drmFamilies')),
3591                 'tbr': tbr,
3592                 'url': fmt_url,
3593                 'width': int_or_none(fmt.get('width')),
3594                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3595                                           'desc' if language_preference < -1 else ''),
3596                 'language_preference': language_preference,
3597                 # Strictly de-prioritize damaged and 3gp formats
3598                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3599             }
3600             mime_mobj = re.match(
3601                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3602             if mime_mobj:
3603                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3604                 dct.update(parse_codecs(mime_mobj.group(2)))
3605             no_audio = dct.get('acodec') == 'none'
3606             no_video = dct.get('vcodec') == 'none'
3607             if no_audio:
3608                 dct['vbr'] = tbr
3609             if no_video:
3610                 dct['abr'] = tbr
3611             if no_audio or no_video:
3612                 dct['downloader_options'] = {
3613                     # Youtube throttles chunks >~10M
3614                     'http_chunk_size': 10485760,
3615                 }
3616                 if dct.get('ext'):
3617                     dct['container'] = dct['ext'] + '_dash'
3618             yield dct
3619
3620         needs_live_processing = self._needs_live_processing(live_status, duration)
3621         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3622
3623         skip_manifests = set(self._configuration_arg('skip'))
3624         if (not self.get_param('youtube_include_hls_manifest', True)
3625                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3626                 or needs_live_processing and skip_bad_formats):
3627             skip_manifests.add('hls')
3628
3629         if not self.get_param('youtube_include_dash_manifest', True):
3630             skip_manifests.add('dash')
3631         if self._configuration_arg('include_live_dash'):
3632             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3633                                                 'Use include_incomplete_formats extractor argument instead')
3634         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3635             skip_manifests.add('dash')
3636
3637         def process_manifest_format(f, proto, itag):
3638             if itag in itags:
3639                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3640                     return False
3641                 itag = f'{itag}-{proto}'
3642             if itag:
3643                 f['format_id'] = itag
3644                 itags[itag] = proto
3645
3646             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3647             if f['quality'] == -1 and f.get('height'):
3648                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3649             return True
3650
3651         subtitles = {}
3652         for sd in streaming_data:
3653             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3654             if hls_manifest_url:
3655                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3656                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3657                 subtitles = self._merge_subtitles(subs, subtitles)
3658                 for f in fmts:
3659                     if process_manifest_format(f, 'hls', self._search_regex(
3660                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3661                         yield f
3662
3663             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3664             if dash_manifest_url:
3665                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3666                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3667                 for f in formats:
3668                     if process_manifest_format(f, 'dash', f['format_id']):
3669                         f['filesize'] = int_or_none(self._search_regex(
3670                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3671                         if needs_live_processing:
3672                             f['is_from_start'] = True
3673
3674                         yield f
3675         yield subtitles
3676
3677     def _extract_storyboard(self, player_responses, duration):
3678         spec = get_first(
3679             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3680         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3681         if not base_url:
3682             return
3683         L = len(spec) - 1
3684         for i, args in enumerate(spec):
3685             args = args.split('#')
3686             counts = list(map(int_or_none, args[:5]))
3687             if len(args) != 8 or not all(counts):
3688                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3689                 continue
3690             width, height, frame_count, cols, rows = counts
3691             N, sigh = args[6:]
3692
3693             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3694             fragment_count = frame_count / (cols * rows)
3695             fragment_duration = duration / fragment_count
3696             yield {
3697                 'format_id': f'sb{i}',
3698                 'format_note': 'storyboard',
3699                 'ext': 'mhtml',
3700                 'protocol': 'mhtml',
3701                 'acodec': 'none',
3702                 'vcodec': 'none',
3703                 'url': url,
3704                 'width': width,
3705                 'height': height,
3706                 'fps': frame_count / duration,
3707                 'rows': rows,
3708                 'columns': cols,
3709                 'fragments': [{
3710                     'url': url.replace('$M', str(j)),
3711                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3712                 } for j in range(math.ceil(fragment_count))],
3713             }
3714
3715     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3716         webpage = None
3717         if 'webpage' not in self._configuration_arg('player_skip'):
3718             query = {'bpctr': '9999999999', 'has_verified': '1'}
3719             if smuggled_data.get('is_story'):
3720                 query['pp'] = self._STORY_PLAYER_PARAMS
3721             webpage = self._download_webpage(
3722                 webpage_url, video_id, fatal=False, query=query)
3723
3724         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3725
3726         player_responses, player_url = self._extract_player_responses(
3727             self._get_requested_clients(url, smuggled_data),
3728             video_id, webpage, master_ytcfg, smuggled_data)
3729
3730         return webpage, master_ytcfg, player_responses, player_url
3731
3732     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3733         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3734         is_live = get_first(video_details, 'isLive')
3735         if is_live is None:
3736             is_live = get_first(live_broadcast_details, 'isLiveNow')
3737         live_content = get_first(video_details, 'isLiveContent')
3738         is_upcoming = get_first(video_details, 'isUpcoming')
3739         post_live = get_first(video_details, 'isPostLiveDvr')
3740         live_status = ('post_live' if post_live
3741                        else 'is_live' if is_live
3742                        else 'is_upcoming' if is_upcoming
3743                        else 'was_live' if live_content
3744                        else 'not_live' if False in (is_live, live_content)
3745                        else None)
3746         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3747         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3748
3749         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3750
3751     def _real_extract(self, url):
3752         url, smuggled_data = unsmuggle_url(url, {})
3753         video_id = self._match_id(url)
3754
3755         base_url = self.http_scheme() + '//www.youtube.com/'
3756         webpage_url = base_url + 'watch?v=' + video_id
3757
3758         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3759
3760         playability_statuses = traverse_obj(
3761             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3762
3763         trailer_video_id = get_first(
3764             playability_statuses,
3765             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3766             expected_type=str)
3767         if trailer_video_id:
3768             return self.url_result(
3769                 trailer_video_id, self.ie_key(), trailer_video_id)
3770
3771         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3772                        if webpage else (lambda x: None))
3773
3774         video_details = traverse_obj(
3775             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3776         microformats = traverse_obj(
3777             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3778             expected_type=dict, default=[])
3779
3780         translated_title = self._get_text(microformats, (..., 'title'))
3781         video_title = (self._preferred_lang and translated_title
3782                        or get_first(video_details, 'title')  # primary
3783                        or translated_title
3784                        or search_meta(['og:title', 'twitter:title', 'title']))
3785         translated_description = self._get_text(microformats, (..., 'description'))
3786         original_description = get_first(video_details, 'shortDescription')
3787         video_description = (
3788             self._preferred_lang and translated_description
3789             # If original description is blank, it will be an empty string.
3790             # Do not prefer translated description in this case.
3791             or original_description if original_description is not None else translated_description)
3792
3793         multifeed_metadata_list = get_first(
3794             player_responses,
3795             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3796             expected_type=str)
3797         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3798             if self.get_param('noplaylist'):
3799                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3800             else:
3801                 entries = []
3802                 feed_ids = []
3803                 for feed in multifeed_metadata_list.split(','):
3804                     # Unquote should take place before split on comma (,) since textual
3805                     # fields may contain comma as well (see
3806                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3807                     feed_data = urllib.parse.parse_qs(
3808                         urllib.parse.unquote_plus(feed))
3809
3810                     def feed_entry(name):
3811                         return try_get(
3812                             feed_data, lambda x: x[name][0], str)
3813
3814                     feed_id = feed_entry('id')
3815                     if not feed_id:
3816                         continue
3817                     feed_title = feed_entry('title')
3818                     title = video_title
3819                     if feed_title:
3820                         title += ' (%s)' % feed_title
3821                     entries.append({
3822                         '_type': 'url_transparent',
3823                         'ie_key': 'Youtube',
3824                         'url': smuggle_url(
3825                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3826                             {'force_singlefeed': True}),
3827                         'title': title,
3828                     })
3829                     feed_ids.append(feed_id)
3830                 self.to_screen(
3831                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3832                     % (', '.join(feed_ids), video_id))
3833                 return self.playlist_result(
3834                     entries, video_id, video_title, video_description)
3835
3836         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
3837                     or int_or_none(get_first(microformats, 'lengthSeconds'))
3838                     or parse_duration(search_meta('duration')) or None)
3839
3840         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
3841             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
3842         if live_status == 'post_live':
3843             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
3844
3845         if not formats:
3846             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3847                 self.report_drm(video_id)
3848             pemr = get_first(
3849                 playability_statuses,
3850                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3851             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3852             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3853             if subreason:
3854                 if subreason == 'The uploader has not made this video available in your country.':
3855                     countries = get_first(microformats, 'availableCountries')
3856                     if not countries:
3857                         regions_allowed = search_meta('regionsAllowed')
3858                         countries = regions_allowed.split(',') if regions_allowed else None
3859                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3860                 reason += f'. {subreason}'
3861             if reason:
3862                 self.raise_no_formats(reason, expected=True)
3863
3864         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3865         if not keywords and webpage:
3866             keywords = [
3867                 unescapeHTML(m.group('content'))
3868                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3869         for keyword in keywords:
3870             if keyword.startswith('yt:stretch='):
3871                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3872                 if mobj:
3873                     # NB: float is intentional for forcing float division
3874                     w, h = (float(v) for v in mobj.groups())
3875                     if w > 0 and h > 0:
3876                         ratio = w / h
3877                         for f in formats:
3878                             if f.get('vcodec') != 'none':
3879                                 f['stretched_ratio'] = ratio
3880                         break
3881         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3882         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3883         if thumbnail_url:
3884             thumbnails.append({
3885                 'url': thumbnail_url,
3886             })
3887         original_thumbnails = thumbnails.copy()
3888
3889         # The best resolution thumbnails sometimes does not appear in the webpage
3890         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3891         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3892         thumbnail_names = [
3893             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3894             # in resolution, these are not the custom thumbnail. So de-prioritize them
3895             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3896             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3897         ]
3898         n_thumbnail_names = len(thumbnail_names)
3899         thumbnails.extend({
3900             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3901                 video_id=video_id, name=name, ext=ext,
3902                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
3903         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3904         for thumb in thumbnails:
3905             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3906             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3907         self._remove_duplicate_formats(thumbnails)
3908         self._downloader._sort_thumbnails(original_thumbnails)
3909
3910         category = get_first(microformats, 'category') or search_meta('genre')
3911         channel_id = str_or_none(
3912             get_first(video_details, 'channelId')
3913             or get_first(microformats, 'externalChannelId')
3914             or search_meta('channelId'))
3915         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3916
3917         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3918         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3919         if not duration and live_end_time and live_start_time:
3920             duration = live_end_time - live_start_time
3921
3922         needs_live_processing = self._needs_live_processing(live_status, duration)
3923
3924         def is_bad_format(fmt):
3925             if needs_live_processing and not fmt.get('is_from_start'):
3926                 return True
3927             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
3928                     and fmt.get('protocol') == 'http_dash_segments'):
3929                 return True
3930
3931         for fmt in filter(is_bad_format, formats):
3932             fmt['preference'] = (fmt.get('preference') or -1) - 10
3933             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
3934
3935         if needs_live_processing:
3936             self._prepare_live_from_start_formats(
3937                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
3938
3939         formats.extend(self._extract_storyboard(player_responses, duration))
3940
3941         # source_preference is lower for throttled/potentially damaged formats
3942         self._sort_formats(formats, (
3943             'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto'))
3944
3945         info = {
3946             'id': video_id,
3947             'title': video_title,
3948             'formats': formats,
3949             'thumbnails': thumbnails,
3950             # The best thumbnail that we are sure exists. Prevents unnecessary
3951             # URL checking if user don't care about getting the best possible thumbnail
3952             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3953             'description': video_description,
3954             'uploader': get_first(video_details, 'author'),
3955             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3956             'uploader_url': owner_profile_url,
3957             'channel_id': channel_id,
3958             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3959             'duration': duration,
3960             'view_count': int_or_none(
3961                 get_first((video_details, microformats), (..., 'viewCount'))
3962                 or search_meta('interactionCount')),
3963             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3964             'age_limit': 18 if (
3965                 get_first(microformats, 'isFamilySafe') is False
3966                 or search_meta('isFamilyFriendly') == 'false'
3967                 or search_meta('og:restrictions:age') == '18+') else 0,
3968             'webpage_url': webpage_url,
3969             'categories': [category] if category else None,
3970             'tags': keywords,
3971             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3972             'live_status': live_status,
3973             'release_timestamp': live_start_time,
3974         }
3975
3976         subtitles = {}
3977         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3978         if pctr:
3979             def get_lang_code(track):
3980                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3981                         or track.get('languageCode'))
3982
3983             # Converted into dicts to remove duplicates
3984             captions = {
3985                 get_lang_code(sub): sub
3986                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3987             translation_languages = {
3988                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3989                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3990
3991             def process_language(container, base_url, lang_code, sub_name, query):
3992                 lang_subs = container.setdefault(lang_code, [])
3993                 for fmt in self._SUBTITLE_FORMATS:
3994                     query.update({
3995                         'fmt': fmt,
3996                     })
3997                     lang_subs.append({
3998                         'ext': fmt,
3999                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
4000                         'name': sub_name,
4001                     })
4002
4003             # NB: Constructing the full subtitle dictionary is slow
4004             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
4005                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
4006             for lang_code, caption_track in captions.items():
4007                 base_url = caption_track.get('baseUrl')
4008                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
4009                 if not base_url:
4010                     continue
4011                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
4012                 if caption_track.get('kind') != 'asr':
4013                     if not lang_code:
4014                         continue
4015                     process_language(
4016                         subtitles, base_url, lang_code, lang_name, {})
4017                     if not caption_track.get('isTranslatable'):
4018                         continue
4019                 for trans_code, trans_name in translation_languages.items():
4020                     if not trans_code:
4021                         continue
4022                     orig_trans_code = trans_code
4023                     if caption_track.get('kind') != 'asr':
4024                         if not get_translated_subs:
4025                             continue
4026                         trans_code += f'-{lang_code}'
4027                         trans_name += format_field(lang_name, None, ' from %s')
4028                     # Add an "-orig" label to the original language so that it can be distinguished.
4029                     # The subs are returned without "-orig" as well for compatibility
4030                     if lang_code == f'a-{orig_trans_code}':
4031                         process_language(
4032                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
4033                     # Setting tlang=lang returns damaged subtitles.
4034                     process_language(automatic_captions, base_url, trans_code, trans_name,
4035                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
4036
4037         info['automatic_captions'] = automatic_captions
4038         info['subtitles'] = subtitles
4039
4040         parsed_url = urllib.parse.urlparse(url)
4041         for component in [parsed_url.fragment, parsed_url.query]:
4042             query = urllib.parse.parse_qs(component)
4043             for k, v in query.items():
4044                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4045                     d_k += '_time'
4046                     if d_k not in info and k in s_ks:
4047                         info[d_k] = parse_duration(query[k][0])
4048
4049         # Youtube Music Auto-generated description
4050         if video_description:
4051             mobj = re.search(
4052                 r'''(?xs)
4053                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4054                     (?P<album>[^\n]+)
4055                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4056                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4057                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4058                     .+\nAuto-generated\ by\ YouTube\.\s*$
4059                 ''', video_description)
4060             if mobj:
4061                 release_year = mobj.group('release_year')
4062                 release_date = mobj.group('release_date')
4063                 if release_date:
4064                     release_date = release_date.replace('-', '')
4065                     if not release_year:
4066                         release_year = release_date[:4]
4067                 info.update({
4068                     'album': mobj.group('album'.strip()),
4069                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4070                     'track': mobj.group('track').strip(),
4071                     'release_date': release_date,
4072                     'release_year': int_or_none(release_year),
4073                 })
4074
4075         initial_data = None
4076         if webpage:
4077             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4078         if not initial_data:
4079             query = {'videoId': video_id}
4080             query.update(self._get_checkok_params())
4081             initial_data = self._extract_response(
4082                 item_id=video_id, ep='next', fatal=False,
4083                 ytcfg=master_ytcfg, query=query,
4084                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4085                 note='Downloading initial data API JSON')
4086
4087         info['comment_count'] = traverse_obj(initial_data, (
4088             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4089             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
4090         ), (
4091             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4092             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
4093         ), expected_type=int_or_none, get_all=False)
4094
4095         try:  # This will error if there is no livechat
4096             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4097         except (KeyError, IndexError, TypeError):
4098             pass
4099         else:
4100             info.setdefault('subtitles', {})['live_chat'] = [{
4101                 # url is needed to set cookies
4102                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4103                 'video_id': video_id,
4104                 'ext': 'json',
4105                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4106                              else 'youtube_live_chat_replay'),
4107             }]
4108
4109         if initial_data:
4110             info['chapters'] = (
4111                 self._extract_chapters_from_json(initial_data, duration)
4112                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4113                 or self._extract_chapters_from_description(video_description, duration)
4114                 or None)
4115
4116         contents = traverse_obj(
4117             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4118             expected_type=list, default=[])
4119
4120         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4121         if vpir:
4122             stl = vpir.get('superTitleLink')
4123             if stl:
4124                 stl = self._get_text(stl)
4125                 if try_get(
4126                         vpir,
4127                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4128                     info['location'] = stl
4129                 else:
4130                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4131                     if mobj:
4132                         info.update({
4133                             'series': mobj.group(1),
4134                             'season_number': int(mobj.group(2)),
4135                             'episode_number': int(mobj.group(3)),
4136                         })
4137             for tlb in (try_get(
4138                     vpir,
4139                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4140                     list) or []):
4141                 tbrs = variadic(
4142                     traverse_obj(
4143                         tlb, 'toggleButtonRenderer',
4144                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer'),
4145                         default=[]))
4146                 for tbr in tbrs:
4147                     for getter, regex in [(
4148                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4149                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4150                                 lambda x: x['accessibility'],
4151                                 lambda x: x['accessibilityData']['accessibilityData'],
4152                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4153                         label = (try_get(tbr, getter, dict) or {}).get('label')
4154                         if label:
4155                             mobj = re.match(regex, label)
4156                             if mobj:
4157                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4158                                 break
4159             sbr_tooltip = try_get(
4160                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4161             if sbr_tooltip:
4162                 like_count, dislike_count = sbr_tooltip.split(' / ')
4163                 info.update({
4164                     'like_count': str_to_int(like_count),
4165                     'dislike_count': str_to_int(dislike_count),
4166                 })
4167             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4168             if vcr:
4169                 vc = self._get_count(vcr, 'viewCount')
4170                 # Upcoming premieres with waiting count are treated as live here
4171                 if vcr.get('isLive'):
4172                     info['concurrent_view_count'] = vc
4173                 elif info.get('view_count') is None:
4174                     info['view_count'] = vc
4175
4176         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4177         if vsir:
4178             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4179             info.update({
4180                 'channel': self._get_text(vor, 'title'),
4181                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4182
4183             rows = try_get(
4184                 vsir,
4185                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4186                 list) or []
4187             multiple_songs = False
4188             for row in rows:
4189                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4190                     multiple_songs = True
4191                     break
4192             for row in rows:
4193                 mrr = row.get('metadataRowRenderer') or {}
4194                 mrr_title = mrr.get('title')
4195                 if not mrr_title:
4196                     continue
4197                 mrr_title = self._get_text(mrr, 'title')
4198                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4199                 if mrr_title == 'License':
4200                     info['license'] = mrr_contents_text
4201                 elif not multiple_songs:
4202                     if mrr_title == 'Album':
4203                         info['album'] = mrr_contents_text
4204                     elif mrr_title == 'Artist':
4205                         info['artist'] = mrr_contents_text
4206                     elif mrr_title == 'Song':
4207                         info['track'] = mrr_contents_text
4208
4209         fallbacks = {
4210             'channel': 'uploader',
4211             'channel_id': 'uploader_id',
4212             'channel_url': 'uploader_url',
4213         }
4214
4215         # The upload date for scheduled, live and past live streams / premieres in microformats
4216         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4217         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4218         upload_date = (
4219             unified_strdate(get_first(microformats, 'uploadDate'))
4220             or unified_strdate(search_meta('uploadDate')))
4221         if not upload_date or (
4222             live_status in ('not_live', None)
4223             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4224         ):
4225             upload_date = strftime_or_none(
4226                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4227         info['upload_date'] = upload_date
4228
4229         for to, frm in fallbacks.items():
4230             if not info.get(to):
4231                 info[to] = info.get(frm)
4232
4233         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4234             v = info.get(s_k)
4235             if v:
4236                 info[d_k] = v
4237
4238         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4239
4240         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4241                       or get_first(video_details, 'isPrivate', expected_type=bool))
4242
4243         info['availability'] = (
4244             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4245             else self._availability(
4246                 is_private=is_private,
4247                 needs_premium=(
4248                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4249                     or False if initial_data and is_private is not None else None),
4250                 needs_subscription=(
4251                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4252                     or False if initial_data and is_private is not None else None),
4253                 needs_auth=info['age_limit'] >= 18,
4254                 is_unlisted=None if is_private is None else (
4255                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4256                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4257
4258         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4259
4260         self.mark_watched(video_id, player_responses)
4261
4262         return info
4263
4264
4265 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4266
4267     @staticmethod
4268     def passthrough_smuggled_data(func):
4269         def _smuggle(entries, smuggled_data):
4270             for entry in entries:
4271                 # TODO: Convert URL to music.youtube instead.
4272                 # Do we need to passthrough any other smuggled_data?
4273                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
4274                 yield entry
4275
4276         @functools.wraps(func)
4277         def wrapper(self, url):
4278             url, smuggled_data = unsmuggle_url(url, {})
4279             if self.is_music_url(url):
4280                 smuggled_data['is_music_url'] = True
4281             info_dict = func(self, url, smuggled_data)
4282             if smuggled_data and info_dict.get('entries'):
4283                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
4284             return info_dict
4285         return wrapper
4286
4287     def _extract_channel_id(self, webpage):
4288         channel_id = self._html_search_meta(
4289             'channelId', webpage, 'channel id', default=None)
4290         if channel_id:
4291             return channel_id
4292         channel_url = self._html_search_meta(
4293             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
4294              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
4295              'twitter:app:url:googleplay'), webpage, 'channel url')
4296         return self._search_regex(
4297             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
4298             channel_url, 'channel id')
4299
4300     @staticmethod
4301     def _extract_basic_item_renderer(item):
4302         # Modified from _extract_grid_item_renderer
4303         known_basic_renderers = (
4304             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4305         )
4306         for key, renderer in item.items():
4307             if not isinstance(renderer, dict):
4308                 continue
4309             elif key in known_basic_renderers:
4310                 return renderer
4311             elif key.startswith('grid') and key.endswith('Renderer'):
4312                 return renderer
4313
4314     def _grid_entries(self, grid_renderer):
4315         for item in grid_renderer['items']:
4316             if not isinstance(item, dict):
4317                 continue
4318             renderer = self._extract_basic_item_renderer(item)
4319             if not isinstance(renderer, dict):
4320                 continue
4321             title = self._get_text(renderer, 'title')
4322
4323             # playlist
4324             playlist_id = renderer.get('playlistId')
4325             if playlist_id:
4326                 yield self.url_result(
4327                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4328                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4329                     video_title=title)
4330                 continue
4331             # video
4332             video_id = renderer.get('videoId')
4333             if video_id:
4334                 yield self._extract_video(renderer)
4335                 continue
4336             # channel
4337             channel_id = renderer.get('channelId')
4338             if channel_id:
4339                 yield self.url_result(
4340                     'https://www.youtube.com/channel/%s' % channel_id,
4341                     ie=YoutubeTabIE.ie_key(), video_title=title)
4342                 continue
4343             # generic endpoint URL support
4344             ep_url = urljoin('https://www.youtube.com/', try_get(
4345                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4346                 str))
4347             if ep_url:
4348                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4349                     if ie.suitable(ep_url):
4350                         yield self.url_result(
4351                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4352                         break
4353
4354     def _music_reponsive_list_entry(self, renderer):
4355         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4356         if video_id:
4357             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4358                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4359         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4360         if playlist_id:
4361             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4362             if video_id:
4363                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4364                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4365             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4366                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4367         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4368         if browse_id:
4369             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4370                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4371
4372     def _shelf_entries_from_content(self, shelf_renderer):
4373         content = shelf_renderer.get('content')
4374         if not isinstance(content, dict):
4375             return
4376         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4377         if renderer:
4378             # TODO: add support for nested playlists so each shelf is processed
4379             # as separate playlist
4380             # TODO: this includes only first N items
4381             yield from self._grid_entries(renderer)
4382         renderer = content.get('horizontalListRenderer')
4383         if renderer:
4384             # TODO
4385             pass
4386
4387     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4388         ep = try_get(
4389             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4390             str)
4391         shelf_url = urljoin('https://www.youtube.com', ep)
4392         if shelf_url:
4393             # Skipping links to another channels, note that checking for
4394             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4395             # will not work
4396             if skip_channels and '/channels?' in shelf_url:
4397                 return
4398             title = self._get_text(shelf_renderer, 'title')
4399             yield self.url_result(shelf_url, video_title=title)
4400         # Shelf may not contain shelf URL, fallback to extraction from content
4401         yield from self._shelf_entries_from_content(shelf_renderer)
4402
4403     def _playlist_entries(self, video_list_renderer):
4404         for content in video_list_renderer['contents']:
4405             if not isinstance(content, dict):
4406                 continue
4407             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4408             if not isinstance(renderer, dict):
4409                 continue
4410             video_id = renderer.get('videoId')
4411             if not video_id:
4412                 continue
4413             yield self._extract_video(renderer)
4414
4415     def _rich_entries(self, rich_grid_renderer):
4416         renderer = traverse_obj(
4417             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4418         video_id = renderer.get('videoId')
4419         if not video_id:
4420             return
4421         yield self._extract_video(renderer)
4422
4423     def _video_entry(self, video_renderer):
4424         video_id = video_renderer.get('videoId')
4425         if video_id:
4426             return self._extract_video(video_renderer)
4427
4428     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4429         url = urljoin('https://youtube.com', traverse_obj(
4430             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4431         if url:
4432             return self.url_result(
4433                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4434
4435     def _post_thread_entries(self, post_thread_renderer):
4436         post_renderer = try_get(
4437             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4438         if not post_renderer:
4439             return
4440         # video attachment
4441         video_renderer = try_get(
4442             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4443         video_id = video_renderer.get('videoId')
4444         if video_id:
4445             entry = self._extract_video(video_renderer)
4446             if entry:
4447                 yield entry
4448         # playlist attachment
4449         playlist_id = try_get(
4450             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4451         if playlist_id:
4452             yield self.url_result(
4453                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4454                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4455         # inline video links
4456         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4457         for run in runs:
4458             if not isinstance(run, dict):
4459                 continue
4460             ep_url = try_get(
4461                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4462             if not ep_url:
4463                 continue
4464             if not YoutubeIE.suitable(ep_url):
4465                 continue
4466             ep_video_id = YoutubeIE._match_id(ep_url)
4467             if video_id == ep_video_id:
4468                 continue
4469             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4470
4471     def _post_thread_continuation_entries(self, post_thread_continuation):
4472         contents = post_thread_continuation.get('contents')
4473         if not isinstance(contents, list):
4474             return
4475         for content in contents:
4476             renderer = content.get('backstagePostThreadRenderer')
4477             if isinstance(renderer, dict):
4478                 yield from self._post_thread_entries(renderer)
4479                 continue
4480             renderer = content.get('videoRenderer')
4481             if isinstance(renderer, dict):
4482                 yield self._video_entry(renderer)
4483
4484     r''' # unused
4485     def _rich_grid_entries(self, contents):
4486         for content in contents:
4487             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4488             if video_renderer:
4489                 entry = self._video_entry(video_renderer)
4490                 if entry:
4491                     yield entry
4492     '''
4493
4494     def _report_history_entries(self, renderer):
4495         for url in traverse_obj(renderer, (
4496                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4497                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4498                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4499             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4500
4501     def _extract_entries(self, parent_renderer, continuation_list):
4502         # continuation_list is modified in-place with continuation_list = [continuation_token]
4503         continuation_list[:] = [None]
4504         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4505         for content in contents:
4506             if not isinstance(content, dict):
4507                 continue
4508             is_renderer = traverse_obj(
4509                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4510                 expected_type=dict)
4511             if not is_renderer:
4512                 if content.get('richItemRenderer'):
4513                     for entry in self._rich_entries(content['richItemRenderer']):
4514                         yield entry
4515                     continuation_list[0] = self._extract_continuation(parent_renderer)
4516                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4517                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4518                     yield from self._report_history_entries(table)
4519                     continuation_list[0] = self._extract_continuation(table)
4520                 continue
4521
4522             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4523             for isr_content in isr_contents:
4524                 if not isinstance(isr_content, dict):
4525                     continue
4526
4527                 known_renderers = {
4528                     'playlistVideoListRenderer': self._playlist_entries,
4529                     'gridRenderer': self._grid_entries,
4530                     'reelShelfRenderer': self._grid_entries,
4531                     'shelfRenderer': self._shelf_entries,
4532                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4533                     'backstagePostThreadRenderer': self._post_thread_entries,
4534                     'videoRenderer': lambda x: [self._video_entry(x)],
4535                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4536                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4537                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4538                 }
4539                 for key, renderer in isr_content.items():
4540                     if key not in known_renderers:
4541                         continue
4542                     for entry in known_renderers[key](renderer):
4543                         if entry:
4544                             yield entry
4545                     continuation_list[0] = self._extract_continuation(renderer)
4546                     break
4547
4548             if not continuation_list[0]:
4549                 continuation_list[0] = self._extract_continuation(is_renderer)
4550
4551         if not continuation_list[0]:
4552             continuation_list[0] = self._extract_continuation(parent_renderer)
4553
4554     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4555         continuation_list = [None]
4556         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4557         tab_content = try_get(tab, lambda x: x['content'], dict)
4558         if not tab_content:
4559             return
4560         parent_renderer = (
4561             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4562             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4563         yield from extract_entries(parent_renderer)
4564         continuation = continuation_list[0]
4565
4566         for page_num in itertools.count(1):
4567             if not continuation:
4568                 break
4569             headers = self.generate_api_headers(
4570                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4571             response = self._extract_response(
4572                 item_id=f'{item_id} page {page_num}',
4573                 query=continuation, headers=headers, ytcfg=ytcfg,
4574                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4575
4576             if not response:
4577                 break
4578             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4579             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4580             visitor_data = self._extract_visitor_data(response) or visitor_data
4581
4582             known_renderers = {
4583                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4584                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4585                 'gridVideoRenderer': (self._grid_entries, 'items'),
4586                 'gridChannelRenderer': (self._grid_entries, 'items'),
4587                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4588                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4589                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4590                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4591                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4592                 'playlistVideoListContinuation': (self._playlist_entries, None),
4593                 'gridContinuation': (self._grid_entries, None),
4594                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4595                 'sectionListContinuation': (extract_entries, None),  # for feeds
4596             }
4597
4598             continuation_items = traverse_obj(response, (
4599                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4600                 'appendContinuationItemsAction', 'continuationItems'
4601             ), 'continuationContents', get_all=False)
4602             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4603
4604             video_items_renderer = None
4605             for key in continuation_item.keys():
4606                 if key not in known_renderers:
4607                     continue
4608                 func, parent_key = known_renderers[key]
4609                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4610                 continuation_list = [None]
4611                 yield from func(video_items_renderer)
4612                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4613
4614             if not video_items_renderer:
4615                 break
4616
4617     @staticmethod
4618     def _extract_selected_tab(tabs, fatal=True):
4619         for tab_renderer in tabs:
4620             if tab_renderer.get('selected'):
4621                 return tab_renderer
4622         if fatal:
4623             raise ExtractorError('Unable to find selected tab')
4624
4625     @staticmethod
4626     def _extract_tab_renderers(response):
4627         return traverse_obj(
4628             response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
4629
4630     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4631         playlist_id = title = description = channel_url = channel_name = channel_id = None
4632         tags = []
4633
4634         selected_tab = self._extract_selected_tab(tabs)
4635         # Deprecated - remove when layout discontinued
4636         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4637         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
4638         metadata_renderer = try_get(
4639             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4640         if metadata_renderer:
4641             channel_name = metadata_renderer.get('title')
4642             channel_url = metadata_renderer.get('channelUrl')
4643             channel_id = metadata_renderer.get('externalId')
4644         else:
4645             metadata_renderer = try_get(
4646                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4647
4648         if metadata_renderer:
4649             title = metadata_renderer.get('title')
4650             description = metadata_renderer.get('description', '')
4651             playlist_id = channel_id
4652             tags = metadata_renderer.get('keywords', '').split()
4653
4654         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4655         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4656         def _get_uncropped(url):
4657             return url_or_none((url or '').split('=')[0] + '=s0')
4658
4659         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
4660         if avatar_thumbnails:
4661             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4662             if uncropped_avatar:
4663                 avatar_thumbnails.append({
4664                     'url': uncropped_avatar,
4665                     'id': 'avatar_uncropped',
4666                     'preference': 1
4667                 })
4668
4669         channel_banners = self._extract_thumbnails(
4670             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4671         for banner in channel_banners:
4672             banner['preference'] = -10
4673
4674         if channel_banners:
4675             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4676             if uncropped_banner:
4677                 channel_banners.append({
4678                     'url': uncropped_banner,
4679                     'id': 'banner_uncropped',
4680                     'preference': -5
4681                 })
4682
4683         # Deprecated - remove when old layout is discontinued
4684         primary_thumbnails = self._extract_thumbnails(
4685             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4686
4687         playlist_thumbnails = self._extract_thumbnails(
4688             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
4689
4690         if playlist_id is None:
4691             playlist_id = item_id
4692
4693         # Deprecated - remove primary_sidebar_renderer when old layout discontinued
4694         # Playlist stats is a text runs array containing [video count, view count, last updated].
4695         # last updated or (view count and last updated) may be missing.
4696         playlist_stats = get_first(
4697             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'),))
4698         last_updated_unix = self._parse_time_text(
4699             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
4700             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
4701
4702         view_count = self._get_count(playlist_stats, 1)
4703         if view_count is None:
4704             view_count = self._get_count(playlist_header_renderer, 'viewCountText')
4705
4706         playlist_count = self._get_count(playlist_stats, 0)
4707         if playlist_count is None:
4708             playlist_count = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
4709
4710         if title is None:
4711             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4712         title += format_field(selected_tab, 'title', ' - %s')
4713         title += format_field(selected_tab, 'expandedText', ' - %s')
4714
4715         metadata = {
4716             'playlist_id': playlist_id,
4717             'playlist_title': title,
4718             'playlist_description': description,
4719             'uploader': channel_name,
4720             'uploader_id': channel_id,
4721             'uploader_url': channel_url,
4722             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
4723             'tags': tags,
4724             'view_count': view_count,
4725             'availability': self._extract_availability(data),
4726             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4727             'playlist_count': playlist_count,
4728             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4729         }
4730         if not channel_id:
4731             owner = traverse_obj(playlist_header_renderer, 'ownerText')
4732             if not owner:
4733                 # Deprecated
4734                 owner = traverse_obj(
4735                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
4736                     ('videoOwner', 'videoOwnerRenderer', 'title'))
4737             owner_text = self._get_text(owner)
4738             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
4739             metadata.update(filter_dict({
4740                 'uploader': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
4741                 'uploader_id': browse_ep.get('browseId'),
4742                 'uploader_url': urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))
4743             }))
4744
4745         metadata.update({
4746             'channel': metadata['uploader'],
4747             'channel_id': metadata['uploader_id'],
4748             'channel_url': metadata['uploader_url']})
4749         return self.playlist_result(
4750             self._entries(
4751                 selected_tab, playlist_id, ytcfg,
4752                 self._extract_account_syncid(ytcfg, data),
4753                 self._extract_visitor_data(data, ytcfg)),
4754             **metadata)
4755
4756     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4757         first_id = last_id = response = None
4758         for page_num in itertools.count(1):
4759             videos = list(self._playlist_entries(playlist))
4760             if not videos:
4761                 return
4762             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4763             if start >= len(videos):
4764                 return
4765             yield from videos[start:]
4766             first_id = first_id or videos[0]['id']
4767             last_id = videos[-1]['id']
4768             watch_endpoint = try_get(
4769                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4770             headers = self.generate_api_headers(
4771                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4772                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4773             query = {
4774                 'playlistId': playlist_id,
4775                 'videoId': watch_endpoint.get('videoId') or last_id,
4776                 'index': watch_endpoint.get('index') or len(videos),
4777                 'params': watch_endpoint.get('params') or 'OAE%3D'
4778             }
4779             response = self._extract_response(
4780                 item_id='%s page %d' % (playlist_id, page_num),
4781                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4782                 check_get_keys='contents'
4783             )
4784             playlist = try_get(
4785                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4786
4787     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4788         title = playlist.get('title') or try_get(
4789             data, lambda x: x['titleText']['simpleText'], str)
4790         playlist_id = playlist.get('playlistId') or item_id
4791
4792         # Delegating everything except mix playlists to regular tab-based playlist URL
4793         playlist_url = urljoin(url, try_get(
4794             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4795             str))
4796
4797         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4798         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4799         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4800
4801         if playlist_url and playlist_url != url and not is_known_unviewable:
4802             return self.url_result(
4803                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4804                 video_title=title)
4805
4806         return self.playlist_result(
4807             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4808             playlist_id=playlist_id, playlist_title=title)
4809
4810     def _extract_availability(self, data):
4811         """
4812         Gets the availability of a given playlist/tab.
4813         Note: Unless YouTube tells us explicitly, we do not assume it is public
4814         @param data: response
4815         """
4816         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4817         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
4818         player_header_privacy = playlist_header_renderer.get('privacy')
4819
4820         badges = self._extract_badges(sidebar_renderer)
4821
4822         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4823         privacy_setting_icon = get_first(
4824             (playlist_header_renderer, sidebar_renderer),
4825             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
4826              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
4827             expected_type=str)
4828
4829         microformats_is_unlisted = traverse_obj(
4830             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
4831
4832         return (
4833             'public' if (
4834                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4835                 or player_header_privacy == 'PUBLIC'
4836                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
4837             else self._availability(
4838                 is_private=(
4839                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4840                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
4841                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
4842                 is_unlisted=(
4843                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4844                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
4845                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
4846                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
4847                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
4848                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
4849                 needs_auth=False))
4850
4851     @staticmethod
4852     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4853         sidebar_renderer = try_get(
4854             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4855         for item in sidebar_renderer:
4856             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4857             if renderer:
4858                 return renderer
4859
4860     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4861         """
4862         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
4863         """
4864         is_playlist = bool(traverse_obj(
4865             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
4866         if not is_playlist:
4867             return
4868         headers = self.generate_api_headers(
4869             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4870             visitor_data=self._extract_visitor_data(data, ytcfg))
4871         query = {
4872             'params': 'wgYCCAA=',
4873             'browseId': f'VL{item_id}'
4874         }
4875         return self._extract_response(
4876             item_id=item_id, headers=headers, query=query,
4877             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4878             note='Redownloading playlist API JSON with unavailable videos')
4879
4880     @functools.cached_property
4881     def skip_webpage(self):
4882         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4883
4884     def _extract_webpage(self, url, item_id, fatal=True):
4885         webpage, data = None, None
4886         for retry in self.RetryManager(fatal=fatal):
4887             try:
4888                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
4889                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4890             except ExtractorError as e:
4891                 if isinstance(e.cause, network_exceptions):
4892                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4893                         retry.error = e
4894                         continue
4895                 self._error_or_warning(e, fatal=fatal)
4896                 break
4897
4898             try:
4899                 self._extract_and_report_alerts(data)
4900             except ExtractorError as e:
4901                 self._error_or_warning(e, fatal=fatal)
4902                 break
4903
4904             # Sometimes youtube returns a webpage with incomplete ytInitialData
4905             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4906             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
4907                 retry.error = ExtractorError('Incomplete yt initial data received')
4908                 continue
4909
4910         return webpage, data
4911
4912     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4913         """Use if failed to extract ytcfg (and data) from initial webpage"""
4914         if not ytcfg and self.is_authenticated:
4915             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4916             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4917                 raise ExtractorError(
4918                     f'{msg}. If you are not downloading private content, or '
4919                     'your cookies are only for the first account and channel,'
4920                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4921                     expected=True)
4922             self.report_warning(msg, only_once=True)
4923
4924     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4925         data = None
4926         if not self.skip_webpage:
4927             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4928             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4929             # Reject webpage data if redirected to home page without explicitly requesting
4930             selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
4931             if (url != 'https://www.youtube.com/feed/recommended'
4932                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4933                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4934                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4935                 if fatal:
4936                     raise ExtractorError(msg, expected=True)
4937                 self.report_warning(msg, only_once=True)
4938         if not data:
4939             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4940             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4941         return data, ytcfg
4942
4943     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4944         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4945         resolve_response = self._extract_response(
4946             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4947             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4948         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4949         for ep_key, ep in endpoints.items():
4950             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4951             if params:
4952                 return self._extract_response(
4953                     item_id=item_id, query=params, ep=ep, headers=headers,
4954                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4955                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4956         err_note = 'Failed to resolve url (does the playlist exist?)'
4957         if fatal:
4958             raise ExtractorError(err_note, expected=True)
4959         self.report_warning(err_note, item_id)
4960
4961     _SEARCH_PARAMS = None
4962
4963     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4964         data = {'query': query}
4965         if params is NO_DEFAULT:
4966             params = self._SEARCH_PARAMS
4967         if params:
4968             data['params'] = params
4969
4970         content_keys = (
4971             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4972             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4973             # ytmusic search
4974             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4975             ('continuationContents', ),
4976         )
4977         display_id = f'query "{query}"'
4978         check_get_keys = tuple({keys[0] for keys in content_keys})
4979         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4980         self._report_playlist_authcheck(ytcfg, fatal=False)
4981
4982         continuation_list = [None]
4983         search = None
4984         for page_num in itertools.count(1):
4985             data.update(continuation_list[0] or {})
4986             headers = self.generate_api_headers(
4987                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4988             search = self._extract_response(
4989                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4990                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4991             slr_contents = traverse_obj(search, *content_keys)
4992             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4993             if not continuation_list[0]:
4994                 break
4995
4996
4997 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4998     IE_DESC = 'YouTube Tabs'
4999     _VALID_URL = r'''(?x:
5000         https?://
5001             (?:\w+\.)?
5002             (?:
5003                 youtube(?:kids)?\.com|
5004                 %(invidious)s
5005             )/
5006             (?:
5007                 (?P<channel_type>channel|c|user|browse)/|
5008                 (?P<not_channel>
5009                     feed/|hashtag/|
5010                     (?:playlist|watch)\?.*?\blist=
5011                 )|
5012                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
5013             )
5014             (?P<id>[^/?\#&]+)
5015     )''' % {
5016         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
5017         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5018     }
5019     IE_NAME = 'youtube:tab'
5020
5021     _TESTS = [{
5022         'note': 'playlists, multipage',
5023         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
5024         'playlist_mincount': 94,
5025         'info_dict': {
5026             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5027             'title': 'Igor Kleiner - Playlists',
5028             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5029             'uploader': 'Igor Kleiner',
5030             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5031             'channel': 'Igor Kleiner',
5032             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5033             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5034             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5035             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5036             'channel_follower_count': int
5037         },
5038     }, {
5039         'note': 'playlists, multipage, different order',
5040         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5041         'playlist_mincount': 94,
5042         'info_dict': {
5043             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5044             'title': 'Igor Kleiner - Playlists',
5045             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5046             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5047             'uploader': 'Igor Kleiner',
5048             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5049             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5050             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5051             'channel': 'Igor Kleiner',
5052             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5053             'channel_follower_count': int
5054         },
5055     }, {
5056         'note': 'playlists, series',
5057         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5058         'playlist_mincount': 5,
5059         'info_dict': {
5060             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5061             'title': '3Blue1Brown - Playlists',
5062             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5063             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5064             'uploader': '3Blue1Brown',
5065             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5066             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5067             'channel': '3Blue1Brown',
5068             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5069             'tags': ['Mathematics'],
5070             'channel_follower_count': int
5071         },
5072     }, {
5073         'note': 'playlists, singlepage',
5074         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5075         'playlist_mincount': 4,
5076         'info_dict': {
5077             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5078             'title': 'ThirstForScience - Playlists',
5079             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5080             'uploader': 'ThirstForScience',
5081             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5082             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5083             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5084             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5085             'tags': 'count:13',
5086             'channel': 'ThirstForScience',
5087             'channel_follower_count': int
5088         }
5089     }, {
5090         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5091         'only_matching': True,
5092     }, {
5093         'note': 'basic, single video playlist',
5094         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5095         'info_dict': {
5096             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5097             'uploader': 'Sergey M.',
5098             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5099             'title': 'youtube-dl public playlist',
5100             'description': '',
5101             'tags': [],
5102             'view_count': int,
5103             'modified_date': '20201130',
5104             'channel': 'Sergey M.',
5105             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5106             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5107             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5108             'availability': 'public',
5109         },
5110         'playlist_count': 1,
5111     }, {
5112         'note': 'empty playlist',
5113         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5114         'info_dict': {
5115             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5116             'uploader': 'Sergey M.',
5117             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5118             'title': 'youtube-dl empty playlist',
5119             'tags': [],
5120             'channel': 'Sergey M.',
5121             'description': '',
5122             'modified_date': '20160902',
5123             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5124             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5125             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5126             'availability': 'public',
5127         },
5128         'playlist_count': 0,
5129     }, {
5130         'note': 'Home tab',
5131         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5132         'info_dict': {
5133             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5134             'title': 'lex will - Home',
5135             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5136             'uploader': 'lex will',
5137             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5138             'channel': 'lex will',
5139             'tags': ['bible', 'history', 'prophesy'],
5140             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5141             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5142             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5143             'channel_follower_count': int
5144         },
5145         'playlist_mincount': 2,
5146     }, {
5147         'note': 'Videos tab',
5148         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5149         'info_dict': {
5150             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5151             'title': 'lex will - Videos',
5152             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5153             'uploader': 'lex will',
5154             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5155             'tags': ['bible', 'history', 'prophesy'],
5156             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5157             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5158             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5159             'channel': 'lex will',
5160             'channel_follower_count': int
5161         },
5162         'playlist_mincount': 975,
5163     }, {
5164         'note': 'Videos tab, sorted by popular',
5165         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5166         'info_dict': {
5167             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5168             'title': 'lex will - Videos',
5169             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5170             'uploader': 'lex will',
5171             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5172             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5173             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5174             'channel': 'lex will',
5175             'tags': ['bible', 'history', 'prophesy'],
5176             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5177             'channel_follower_count': int
5178         },
5179         'playlist_mincount': 199,
5180     }, {
5181         'note': 'Playlists tab',
5182         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5183         'info_dict': {
5184             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5185             'title': 'lex will - Playlists',
5186             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5187             'uploader': 'lex will',
5188             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5189             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5190             'channel': 'lex will',
5191             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5192             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5193             'tags': ['bible', 'history', 'prophesy'],
5194             'channel_follower_count': int
5195         },
5196         'playlist_mincount': 17,
5197     }, {
5198         'note': 'Community tab',
5199         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5200         'info_dict': {
5201             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5202             'title': 'lex will - Community',
5203             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5204             'uploader': 'lex will',
5205             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5206             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5207             'channel': 'lex will',
5208             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5209             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5210             'tags': ['bible', 'history', 'prophesy'],
5211             'channel_follower_count': int
5212         },
5213         'playlist_mincount': 18,
5214     }, {
5215         'note': 'Channels tab',
5216         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5217         'info_dict': {
5218             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5219             'title': 'lex will - Channels',
5220             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5221             'uploader': 'lex will',
5222             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5223             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5224             'channel': 'lex will',
5225             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5226             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5227             'tags': ['bible', 'history', 'prophesy'],
5228             'channel_follower_count': int
5229         },
5230         'playlist_mincount': 12,
5231     }, {
5232         'note': 'Search tab',
5233         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5234         'playlist_mincount': 40,
5235         'info_dict': {
5236             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5237             'title': '3Blue1Brown - Search - linear algebra',
5238             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5239             'uploader': '3Blue1Brown',
5240             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5241             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5242             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5243             'tags': ['Mathematics'],
5244             'channel': '3Blue1Brown',
5245             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5246             'channel_follower_count': int
5247         },
5248     }, {
5249         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5250         'only_matching': True,
5251     }, {
5252         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5253         'only_matching': True,
5254     }, {
5255         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5256         'only_matching': True,
5257     }, {
5258         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5259         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5260         'info_dict': {
5261             'title': '29C3: Not my department',
5262             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5263             'uploader': 'Christiaan008',
5264             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5265             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5266             'tags': [],
5267             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5268             'view_count': int,
5269             'modified_date': '20150605',
5270             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5271             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5272             'channel': 'Christiaan008',
5273             'availability': 'public',
5274         },
5275         'playlist_count': 96,
5276     }, {
5277         'note': 'Large playlist',
5278         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5279         'info_dict': {
5280             'title': 'Uploads from Cauchemar',
5281             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5282             'uploader': 'Cauchemar',
5283             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5284             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5285             'tags': [],
5286             'modified_date': r're:\d{8}',
5287             'channel': 'Cauchemar',
5288             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5289             'view_count': int,
5290             'description': '',
5291             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5292             'availability': 'public',
5293         },
5294         'playlist_mincount': 1123,
5295         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5296     }, {
5297         'note': 'even larger playlist, 8832 videos',
5298         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5299         'only_matching': True,
5300     }, {
5301         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5302         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5303         'info_dict': {
5304             'title': 'Uploads from Interstellar Movie',
5305             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5306             'uploader': 'Interstellar Movie',
5307             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5308             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5309             'tags': [],
5310             'view_count': int,
5311             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5312             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5313             'channel': 'Interstellar Movie',
5314             'description': '',
5315             'modified_date': r're:\d{8}',
5316             'availability': 'public',
5317         },
5318         'playlist_mincount': 21,
5319     }, {
5320         'note': 'Playlist with "show unavailable videos" button',
5321         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5322         'info_dict': {
5323             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5324             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5325             'uploader': 'Phim Siêu Nhân Nhật Bản',
5326             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5327             'view_count': int,
5328             'channel': 'Phim Siêu Nhân Nhật Bản',
5329             'tags': [],
5330             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5331             'description': '',
5332             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5333             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5334             'modified_date': r're:\d{8}',
5335             'availability': 'public',
5336         },
5337         'playlist_mincount': 200,
5338         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5339     }, {
5340         'note': 'Playlist with unavailable videos in page 7',
5341         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5342         'info_dict': {
5343             'title': 'Uploads from BlankTV',
5344             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5345             'uploader': 'BlankTV',
5346             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5347             'channel': 'BlankTV',
5348             'channel_url': 'https://www.youtube.com/c/blanktv',
5349             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5350             'view_count': int,
5351             'tags': [],
5352             'uploader_url': 'https://www.youtube.com/c/blanktv',
5353             'modified_date': r're:\d{8}',
5354             'description': '',
5355             'availability': 'public',
5356         },
5357         'playlist_mincount': 1000,
5358         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5359     }, {
5360         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5361         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5362         'info_dict': {
5363             'title': 'Data Analysis with Dr Mike Pound',
5364             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5365             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5366             'uploader': 'Computerphile',
5367             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5368             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5369             'tags': [],
5370             'view_count': int,
5371             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5372             'channel_url': 'https://www.youtube.com/user/Computerphile',
5373             'channel': 'Computerphile',
5374             'availability': 'public',
5375             'modified_date': '20190712',
5376         },
5377         'playlist_mincount': 11,
5378     }, {
5379         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5380         'only_matching': True,
5381     }, {
5382         'note': 'Playlist URL that does not actually serve a playlist',
5383         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5384         'info_dict': {
5385             'id': 'FqZTN594JQw',
5386             'ext': 'webm',
5387             'title': "Smiley's People 01 detective, Adventure Series, Action",
5388             'uploader': 'STREEM',
5389             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5390             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5391             'upload_date': '20150526',
5392             'license': 'Standard YouTube License',
5393             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5394             'categories': ['People & Blogs'],
5395             'tags': list,
5396             'view_count': int,
5397             'like_count': int,
5398         },
5399         'params': {
5400             'skip_download': True,
5401         },
5402         'skip': 'This video is not available.',
5403         'add_ie': [YoutubeIE.ie_key()],
5404     }, {
5405         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5406         'only_matching': True,
5407     }, {
5408         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5409         'only_matching': True,
5410     }, {
5411         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5412         'info_dict': {
5413             'id': 'Wq15eF5vCbI',  # This will keep changing
5414             'ext': 'mp4',
5415             'title': str,
5416             'uploader': 'Sky News',
5417             'uploader_id': 'skynews',
5418             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5419             'upload_date': r're:\d{8}',
5420             'description': str,
5421             'categories': ['News & Politics'],
5422             'tags': list,
5423             'like_count': int,
5424             'release_timestamp': int,
5425             'channel': 'Sky News',
5426             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5427             'age_limit': 0,
5428             'view_count': int,
5429             'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
5430             'playable_in_embed': True,
5431             'release_date': r're:\d+',
5432             'availability': 'public',
5433             'live_status': 'is_live',
5434             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5435             'channel_follower_count': int,
5436             'concurrent_view_count': int,
5437         },
5438         'params': {
5439             'skip_download': True,
5440         },
5441         'expected_warnings': ['Ignoring subtitle tracks found in '],
5442     }, {
5443         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5444         'info_dict': {
5445             'id': 'a48o2S1cPoo',
5446             'ext': 'mp4',
5447             'title': 'The Young Turks - Live Main Show',
5448             'uploader': 'The Young Turks',
5449             'uploader_id': 'TheYoungTurks',
5450             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5451             'upload_date': '20150715',
5452             'license': 'Standard YouTube License',
5453             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5454             'categories': ['News & Politics'],
5455             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5456             'like_count': int,
5457         },
5458         'params': {
5459             'skip_download': True,
5460         },
5461         'only_matching': True,
5462     }, {
5463         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5464         'only_matching': True,
5465     }, {
5466         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5467         'only_matching': True,
5468     }, {
5469         'note': 'A channel that is not live. Should raise error',
5470         'url': 'https://www.youtube.com/user/numberphile/live',
5471         'only_matching': True,
5472     }, {
5473         'url': 'https://www.youtube.com/feed/trending',
5474         'only_matching': True,
5475     }, {
5476         'url': 'https://www.youtube.com/feed/library',
5477         'only_matching': True,
5478     }, {
5479         'url': 'https://www.youtube.com/feed/history',
5480         'only_matching': True,
5481     }, {
5482         'url': 'https://www.youtube.com/feed/subscriptions',
5483         'only_matching': True,
5484     }, {
5485         'url': 'https://www.youtube.com/feed/watch_later',
5486         'only_matching': True,
5487     }, {
5488         'note': 'Recommended - redirects to home page.',
5489         'url': 'https://www.youtube.com/feed/recommended',
5490         'only_matching': True,
5491     }, {
5492         'note': 'inline playlist with not always working continuations',
5493         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5494         'only_matching': True,
5495     }, {
5496         'url': 'https://www.youtube.com/course',
5497         'only_matching': True,
5498     }, {
5499         'url': 'https://www.youtube.com/zsecurity',
5500         'only_matching': True,
5501     }, {
5502         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5503         'only_matching': True,
5504     }, {
5505         'url': 'https://www.youtube.com/TheYoungTurks/live',
5506         'only_matching': True,
5507     }, {
5508         'url': 'https://www.youtube.com/hashtag/cctv9',
5509         'info_dict': {
5510             'id': 'cctv9',
5511             'title': '#cctv9',
5512             'tags': [],
5513         },
5514         'playlist_mincount': 300,  # not consistent but should be over 300
5515     }, {
5516         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5517         'only_matching': True,
5518     }, {
5519         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5520         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5521         'only_matching': True
5522     }, {
5523         'note': '/browse/ should redirect to /channel/',
5524         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5525         'only_matching': True
5526     }, {
5527         'note': 'VLPL, should redirect to playlist?list=PL...',
5528         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5529         'info_dict': {
5530             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5531             'uploader': 'NoCopyrightSounds',
5532             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5533             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5534             'title': 'NCS : All Releases 💿',
5535             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5536             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5537             'modified_date': r're:\d{8}',
5538             'view_count': int,
5539             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5540             'tags': [],
5541             'channel': 'NoCopyrightSounds',
5542             'availability': 'public',
5543         },
5544         'playlist_mincount': 166,
5545         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5546     }, {
5547         'note': 'Topic, should redirect to playlist?list=UU...',
5548         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5549         'info_dict': {
5550             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5551             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5552             'title': 'Uploads from Royalty Free Music - Topic',
5553             'uploader': 'Royalty Free Music - Topic',
5554             'tags': [],
5555             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5556             'channel': 'Royalty Free Music - Topic',
5557             'view_count': int,
5558             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5559             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5560             'modified_date': r're:\d{8}',
5561             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5562             'description': '',
5563             'availability': 'public',
5564         },
5565         'expected_warnings': [
5566             'The URL does not have a videos tab',
5567             r'[Uu]navailable videos (are|will be) hidden',
5568         ],
5569         'playlist_mincount': 101,
5570     }, {
5571         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
5572         # Treat as a general feed
5573         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5574         'info_dict': {
5575             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5576             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5577             'tags': [],
5578         },
5579         'playlist_mincount': 9,
5580     }, {
5581         'note': 'Youtube music Album',
5582         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5583         'info_dict': {
5584             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5585             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5586             'tags': [],
5587             'view_count': int,
5588             'description': '',
5589             'availability': 'unlisted',
5590             'modified_date': r're:\d{8}',
5591         },
5592         'playlist_count': 50,
5593     }, {
5594         'note': 'unlisted single video playlist',
5595         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5596         'info_dict': {
5597             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5598             'uploader': 'colethedj',
5599             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5600             'title': 'yt-dlp unlisted playlist test',
5601             'availability': 'unlisted',
5602             'tags': [],
5603             'modified_date': '20220418',
5604             'channel': 'colethedj',
5605             'view_count': int,
5606             'description': '',
5607             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5608             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5609             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5610         },
5611         'playlist_count': 1,
5612     }, {
5613         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5614         'url': 'https://www.youtube.com/feed/recommended',
5615         'info_dict': {
5616             'id': 'recommended',
5617             'title': 'recommended',
5618             'tags': [],
5619         },
5620         'playlist_mincount': 50,
5621         'params': {
5622             'skip_download': True,
5623             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5624         },
5625     }, {
5626         'note': 'API Fallback: /videos tab, sorted by oldest first',
5627         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5628         'info_dict': {
5629             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5630             'title': 'Cody\'sLab - Videos',
5631             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5632             'uploader': 'Cody\'sLab',
5633             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5634             'channel': 'Cody\'sLab',
5635             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5636             'tags': [],
5637             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5638             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5639             'channel_follower_count': int
5640         },
5641         'playlist_mincount': 650,
5642         'params': {
5643             'skip_download': True,
5644             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5645         },
5646         'skip': 'Query for sorting no longer works',
5647     }, {
5648         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5649         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5650         'info_dict': {
5651             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5652             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5653             'title': 'Uploads from Royalty Free Music - Topic',
5654             'uploader': 'Royalty Free Music - Topic',
5655             'modified_date': r're:\d{8}',
5656             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5657             'description': '',
5658             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5659             'tags': [],
5660             'channel': 'Royalty Free Music - Topic',
5661             'view_count': int,
5662             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5663             'availability': 'public',
5664         },
5665         'playlist_mincount': 101,
5666         'params': {
5667             'skip_download': True,
5668             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5669         },
5670     }, {
5671         'note': 'non-standard redirect to regional channel',
5672         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5673         'only_matching': True
5674     }, {
5675         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5676         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5677         'info_dict': {
5678             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5679             'modified_date': '20220407',
5680             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5681             'tags': [],
5682             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5683             'uploader': 'pukkandan',
5684             'availability': 'unlisted',
5685             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5686             'channel': 'pukkandan',
5687             'description': 'Test for collaborative playlist',
5688             'title': 'yt-dlp test - collaborative playlist',
5689             'view_count': int,
5690             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5691         },
5692         'playlist_mincount': 2
5693     }, {
5694         'note': 'translated tab name',
5695         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5696         'info_dict': {
5697             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5698             'tags': [],
5699             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5700             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5701             'description': 'test description',
5702             'title': 'cole-dlp-test-acc - 再生リスト',
5703             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5704             'uploader': 'cole-dlp-test-acc',
5705             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5706             'channel': 'cole-dlp-test-acc',
5707             'channel_follower_count': int,
5708         },
5709         'playlist_mincount': 1,
5710         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5711         'expected_warnings': ['Preferring "ja"'],
5712     }, {
5713         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5714         'note': 'preferred lang set with playlist with translated video titles',
5715         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5716         'info_dict': {
5717             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5718             'tags': [],
5719             'view_count': int,
5720             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5721             'uploader': 'cole-dlp-test-acc',
5722             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5723             'channel': 'cole-dlp-test-acc',
5724             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5725             'description': 'test',
5726             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5727             'title': 'dlp test playlist',
5728             'availability': 'public',
5729         },
5730         'playlist_mincount': 1,
5731         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5732         'expected_warnings': ['Preferring "ja"'],
5733     }, {
5734         # shorts audio pivot for 2GtVksBMYFM.
5735         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5736         'info_dict': {
5737             'id': 'sfv_audio_pivot',
5738             'title': 'sfv_audio_pivot',
5739             'tags': [],
5740         },
5741         'playlist_mincount': 50,
5742
5743     }, {
5744         # Channel with a real live tab (not to be mistaken with streams tab)
5745         # Do not treat like it should redirect to live stream
5746         'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
5747         'info_dict': {
5748             'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
5749             'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
5750             'tags': [],
5751         },
5752         'playlist_mincount': 20,
5753     }, {
5754         # Tab name is not the same as tab id
5755         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
5756         'info_dict': {
5757             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5758             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
5759             'tags': [],
5760         },
5761         'playlist_mincount': 8,
5762     }, {
5763         # Home tab id is literally home. Not to get mistaken with featured
5764         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
5765         'info_dict': {
5766             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5767             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
5768             'tags': [],
5769         },
5770         'playlist_mincount': 8,
5771     }, {
5772         # Should get three playlists for videos, shorts and streams tabs
5773         'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5774         'info_dict': {
5775             'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5776             'title': 'Uploads for UCK9V2B22uJYu3N7eR_BT9QA'
5777         },
5778         'playlist_count': 3,
5779     }, {
5780         # Shorts tab with channel with handle
5781         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
5782         'info_dict': {
5783             'id': 'UC0intLFzLaudFG-xAvUEO-A',
5784             'title': 'Not Just Bikes - Shorts',
5785             'tags': 'count:12',
5786             'uploader': 'Not Just Bikes',
5787             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5788             'description': 'md5:7513148b1f02b924783157d84c4ea555',
5789             'channel_follower_count': int,
5790             'uploader_id': 'UC0intLFzLaudFG-xAvUEO-A',
5791             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
5792             'uploader_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5793             'channel': 'Not Just Bikes',
5794         },
5795         'playlist_mincount': 10,
5796     }, {
5797         # Streams tab
5798         'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
5799         'info_dict': {
5800             'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5801             'title': '中村悠一 - Live',
5802             'tags': 'count:7',
5803             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5804             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5805             'uploader_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5806             'channel': '中村悠一',
5807             'uploader_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5808             'channel_follower_count': int,
5809             'uploader': '中村悠一',
5810             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
5811         },
5812         'playlist_mincount': 60,
5813     }, {
5814         # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
5815         # See test_youtube_lists
5816         'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
5817         'only_matching': True,
5818     }, {
5819         # No uploads and no UCID given. Should fail with no uploads error
5820         # See test_youtube_lists
5821         'url': 'https://www.youtube.com/news',
5822         'only_matching': True
5823     }, {
5824         # No videos tab but has a shorts tab
5825         'url': 'https://www.youtube.com/c/TKFShorts',
5826         'info_dict': {
5827             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5828             'title': 'Shorts Break - Shorts',
5829             'tags': 'count:32',
5830             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5831             'channel': 'Shorts Break',
5832             'description': 'md5:a6c234cf3d50d878ef8721e34457cd11',
5833             'uploader': 'Shorts Break',
5834             'channel_follower_count': int,
5835             'uploader_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5836             'uploader_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
5837             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
5838         },
5839         'playlist_mincount': 30,
5840     }, {
5841         # Trending Now Tab. tab id is empty
5842         'url': 'https://www.youtube.com/feed/trending',
5843         'info_dict': {
5844             'id': 'trending',
5845             'title': 'trending - Now',
5846             'tags': [],
5847         },
5848         'playlist_mincount': 30,
5849     }, {
5850         # Trending Gaming Tab. tab id is empty
5851         'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
5852         'info_dict': {
5853             'id': 'trending',
5854             'title': 'trending - Gaming',
5855             'tags': [],
5856         },
5857         'playlist_mincount': 30,
5858     }, {
5859         # Shorts url result in shorts tab
5860         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
5861         'info_dict': {
5862             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5863             'title': 'cole-dlp-test-acc - Shorts',
5864             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5865             'channel': 'cole-dlp-test-acc',
5866             'channel_follower_count': int,
5867             'description': 'test description',
5868             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5869             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5870             'tags': [],
5871             'uploader': 'cole-dlp-test-acc',
5872             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5873
5874         },
5875         'playlist': [{
5876             'info_dict': {
5877                 '_type': 'url',
5878                 'ie_key': 'Youtube',
5879                 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
5880                 'id': 'sSM9J5YH_60',
5881                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5882                 'title': 'SHORT short',
5883                 'channel': 'cole-dlp-test-acc',
5884                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5885                 'view_count': int,
5886                 'thumbnails': list,
5887             }
5888         }],
5889         'params': {'extract_flat': True},
5890     }, {
5891         # Live video status should be extracted
5892         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
5893         'info_dict': {
5894             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5895             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live',  # TODO, should be Minecraft - Live or Minecraft - Topic - Live
5896             'tags': []
5897         },
5898         'playlist': [{
5899             'info_dict': {
5900                 '_type': 'url',
5901                 'ie_key': 'Youtube',
5902                 'url': 'startswith:https://www.youtube.com/watch?v=',
5903                 'id': str,
5904                 'title': str,
5905                 'live_status': 'is_live',
5906                 'channel_id': str,
5907                 'channel_url': str,
5908                 'concurrent_view_count': int,
5909                 'channel': str,
5910             }
5911         }],
5912         'params': {'extract_flat': True},
5913         'playlist_mincount': 1
5914     }]
5915
5916     @classmethod
5917     def suitable(cls, url):
5918         return False if YoutubeIE.suitable(url) else super().suitable(url)
5919
5920     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
5921
5922     def _get_url_mobj(self, url):
5923         mobj = self._URL_RE.match(url).groupdict()
5924         mobj.update((k, '') for k, v in mobj.items() if v is None)
5925         return mobj
5926
5927     def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
5928         tab_name = (tab.get('title') or '').lower()
5929         tab_url = urljoin(base_url, traverse_obj(
5930             tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
5931
5932         tab_id = (traverse_obj(tab, 'tabIdentifier', expected_type=str)
5933                   or tab_url and self._get_url_mobj(tab_url)['tab'][1:])
5934         if tab_id:
5935             return tab_id, tab_name
5936
5937         # Fallback to tab name if we cannot get the tab id.
5938         # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
5939         # Note that in the case of translated tab name this may result in an empty string, which we don't want.
5940         self.write_debug(f'Falling back to selected tab name: {tab_name}')
5941         return {
5942             'home': 'featured',
5943             'live': 'streams',
5944         }.get(tab_name, tab_name), tab_name
5945
5946     def _has_tab(self, tabs, tab_id):
5947         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
5948
5949     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5950     def _real_extract(self, url, smuggled_data):
5951         item_id = self._match_id(url)
5952         url = urllib.parse.urlunparse(
5953             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5954         compat_opts = self.get_param('compat_opts', [])
5955
5956         mobj = self._get_url_mobj(url)
5957         pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
5958         if is_channel:
5959             if smuggled_data.get('is_music_url'):
5960                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5961                     item_id = item_id[2:]
5962                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5963                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5964                     mdata = self._extract_tab_endpoint(
5965                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5966                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5967                                         get_all=False, expected_type=str)
5968                     if not murl:
5969                         raise ExtractorError('Failed to resolve album to playlist')
5970                     return self.url_result(murl, YoutubeTabIE)
5971                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5972                     pre = f'https://www.youtube.com/channel/{item_id}'
5973
5974         original_tab_id = tab[1:]
5975         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5976             tab = '/videos'
5977
5978         url = ''.join((pre, tab, post))
5979         mobj = self._get_url_mobj(url)
5980
5981         # Handle both video/playlist URLs
5982         qs = parse_qs(url)
5983         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5984
5985         if not video_id and mobj['not_channel'].startswith('watch'):
5986             if not playlist_id:
5987                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5988                 raise ExtractorError('Unable to recognize tab page')
5989             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5990             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5991             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5992             mobj = self._get_url_mobj(url)
5993
5994         if not self._yes_playlist(playlist_id, video_id):
5995             return self.url_result(
5996                 f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
5997
5998         data, ytcfg = self._extract_data(url, item_id)
5999
6000         # YouTube may provide a non-standard redirect to the regional channel
6001         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
6002         # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
6003         redirect_url = traverse_obj(
6004             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
6005         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
6006             redirect_url = ''.join((
6007                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
6008             self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
6009             return self.url_result(redirect_url, YoutubeTabIE)
6010
6011         tab_results = []
6012         tabs = self._extract_tab_renderers(data)
6013         if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
6014             selected_tab = self._extract_selected_tab(tabs)
6015             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
6016             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
6017
6018             if not original_tab_id and selected_tab_name:
6019                 self.to_screen('Channel URLs download all uploads of the channel. '
6020                                'To download only the videos in a specific tab, pass the tab\'s URL')
6021                 if self._has_tab(tabs, 'streams'):
6022                     tab_results.append(self.url_result(''.join((pre, '/streams', post))))
6023                 if self._has_tab(tabs, 'shorts'):
6024                     tab_results.append(self.url_result(''.join((pre, '/shorts', post))))
6025                 # XXX: Members-only tab should also be extracted
6026
6027                 if not tab_results and selected_tab_id != 'videos':
6028                     # Channel does not have streams, shorts or videos tabs
6029                     if item_id[:2] != 'UC':
6030                         raise ExtractorError('This channel has no uploads', expected=True)
6031
6032                     # Topic channels don't have /videos. Use the equivalent playlist instead
6033                     pl_id = f'UU{item_id[2:]}'
6034                     pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
6035                     try:
6036                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
6037                     except ExtractorError:
6038                         raise ExtractorError('This channel has no uploads', expected=True)
6039                     else:
6040                         item_id, url = pl_id, pl_url
6041                         self.to_screen(
6042                             f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
6043
6044                 elif tab_results and selected_tab_id != 'videos':
6045                     # When there are shorts/live tabs but not videos tab
6046                     url, data = ''.join((pre, post)), None
6047
6048             elif (original_tab_id or 'videos') != selected_tab_id:
6049                 if original_tab_id == 'live':
6050                     # Live tab should have redirected to the video
6051                     # Except in the case the channel has an actual live tab
6052                     # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
6053                     raise UserNotLive(video_id=mobj['id'])
6054                 elif selected_tab_name:
6055                     raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
6056
6057                 # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
6058                 url = f'{pre}{post}'
6059
6060         self.write_debug(f'Final URL: {url}')
6061
6062         # YouTube sometimes provides a button to reload playlist with unavailable videos.
6063         if 'no-youtube-unavailable-videos' not in compat_opts:
6064             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
6065         self._extract_and_report_alerts(data, only_once=True)
6066
6067         tabs = self._extract_tab_renderers(data)
6068         if tabs:
6069             tab_results[:0] = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
6070             tab_results[0].update({
6071                 'extractor_key': YoutubeTabIE.ie_key(),
6072                 'extractor': YoutubeTabIE.IE_NAME,
6073                 'webpage_url': url,
6074             })
6075
6076         if len(tab_results) == 1:
6077             return tab_results[0]
6078         elif len(tab_results) > 1:
6079             return self.playlist_result(tab_results, item_id, title=f'Uploads for {item_id}')
6080
6081         playlist = traverse_obj(
6082             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
6083         if playlist:
6084             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
6085
6086         video_id = traverse_obj(
6087             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
6088         if video_id:
6089             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
6090                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
6091             return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6092
6093         raise ExtractorError('Unable to recognize tab page')
6094
6095
6096 class YoutubePlaylistIE(InfoExtractor):
6097     IE_DESC = 'YouTube playlists'
6098     _VALID_URL = r'''(?x)(?:
6099                         (?:https?://)?
6100                         (?:\w+\.)?
6101                         (?:
6102                             (?:
6103                                 youtube(?:kids)?\.com|
6104                                 %(invidious)s
6105                             )
6106                             /.*?\?.*?\blist=
6107                         )?
6108                         (?P<id>%(playlist_id)s)
6109                      )''' % {
6110         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
6111         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
6112     }
6113     IE_NAME = 'youtube:playlist'
6114     _TESTS = [{
6115         'note': 'issue #673',
6116         'url': 'PLBB231211A4F62143',
6117         'info_dict': {
6118             'title': '[OLD]Team Fortress 2 (Class-based LP)',
6119             'id': 'PLBB231211A4F62143',
6120             'uploader': 'Wickman',
6121             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6122             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
6123             'view_count': int,
6124             'uploader_url': 'https://www.youtube.com/c/WickmanVT',
6125             'modified_date': r're:\d{8}',
6126             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6127             'channel': 'Wickman',
6128             'tags': [],
6129             'channel_url': 'https://www.youtube.com/c/WickmanVT',
6130             'availability': 'public',
6131         },
6132         'playlist_mincount': 29,
6133     }, {
6134         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6135         'info_dict': {
6136             'title': 'YDL_safe_search',
6137             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6138         },
6139         'playlist_count': 2,
6140         'skip': 'This playlist is private',
6141     }, {
6142         'note': 'embedded',
6143         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6144         'playlist_count': 4,
6145         'info_dict': {
6146             'title': 'JODA15',
6147             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6148             'uploader': 'milan',
6149             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6150             'description': '',
6151             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6152             'tags': [],
6153             'modified_date': '20140919',
6154             'view_count': int,
6155             'channel': 'milan',
6156             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6157             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6158             'availability': 'public',
6159         },
6160         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
6161     }, {
6162         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6163         'playlist_mincount': 455,
6164         'info_dict': {
6165             'title': '2018 Chinese New Singles (11/6 updated)',
6166             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6167             'uploader': 'LBK',
6168             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6169             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
6170             'channel': 'LBK',
6171             'view_count': int,
6172             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
6173             'tags': [],
6174             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
6175             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6176             'modified_date': r're:\d{8}',
6177             'availability': 'public',
6178         },
6179         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6180     }, {
6181         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
6182         'only_matching': True,
6183     }, {
6184         # music album playlist
6185         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
6186         'only_matching': True,
6187     }]
6188
6189     @classmethod
6190     def suitable(cls, url):
6191         if YoutubeTabIE.suitable(url):
6192             return False
6193         from ..utils import parse_qs
6194         qs = parse_qs(url)
6195         if qs.get('v', [None])[0]:
6196             return False
6197         return super().suitable(url)
6198
6199     def _real_extract(self, url):
6200         playlist_id = self._match_id(url)
6201         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
6202         url = update_url_query(
6203             'https://www.youtube.com/playlist',
6204             parse_qs(url) or {'list': playlist_id})
6205         if is_music_url:
6206             url = smuggle_url(url, {'is_music_url': True})
6207         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6208
6209
6210 class YoutubeYtBeIE(InfoExtractor):
6211     IE_DESC = 'youtu.be'
6212     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
6213     _TESTS = [{
6214         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
6215         'info_dict': {
6216             'id': 'yeWKywCrFtk',
6217             'ext': 'mp4',
6218             'title': 'Small Scale Baler and Braiding Rugs',
6219             'uploader': 'Backus-Page House Museum',
6220             'uploader_id': 'backuspagemuseum',
6221             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
6222             'upload_date': '20161008',
6223             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
6224             'categories': ['Nonprofits & Activism'],
6225             'tags': list,
6226             'like_count': int,
6227             'age_limit': 0,
6228             'playable_in_embed': True,
6229             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
6230             'channel': 'Backus-Page House Museum',
6231             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
6232             'live_status': 'not_live',
6233             'view_count': int,
6234             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
6235             'availability': 'public',
6236             'duration': 59,
6237             'comment_count': int,
6238             'channel_follower_count': int
6239         },
6240         'params': {
6241             'noplaylist': True,
6242             'skip_download': True,
6243         },
6244     }, {
6245         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6246         'only_matching': True,
6247     }]
6248
6249     def _real_extract(self, url):
6250         mobj = self._match_valid_url(url)
6251         video_id = mobj.group('id')
6252         playlist_id = mobj.group('playlist_id')
6253         return self.url_result(
6254             update_url_query('https://www.youtube.com/watch', {
6255                 'v': video_id,
6256                 'list': playlist_id,
6257                 'feature': 'youtu.be',
6258             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6259
6260
6261 class YoutubeLivestreamEmbedIE(InfoExtractor):
6262     IE_DESC = 'YouTube livestream embeds'
6263     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6264     _TESTS = [{
6265         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6266         'only_matching': True,
6267     }]
6268
6269     def _real_extract(self, url):
6270         channel_id = self._match_id(url)
6271         return self.url_result(
6272             f'https://www.youtube.com/channel/{channel_id}/live',
6273             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6274
6275
6276 class YoutubeYtUserIE(InfoExtractor):
6277     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6278     IE_NAME = 'youtube:user'
6279     _VALID_URL = r'ytuser:(?P<id>.+)'
6280     _TESTS = [{
6281         'url': 'ytuser:phihag',
6282         'only_matching': True,
6283     }]
6284
6285     def _real_extract(self, url):
6286         user_id = self._match_id(url)
6287         return self.url_result(
6288             'https://www.youtube.com/user/%s/videos' % user_id,
6289             ie=YoutubeTabIE.ie_key(), video_id=user_id)
6290
6291
6292 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6293     IE_NAME = 'youtube:favorites'
6294     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6295     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6296     _LOGIN_REQUIRED = True
6297     _TESTS = [{
6298         'url': ':ytfav',
6299         'only_matching': True,
6300     }, {
6301         'url': ':ytfavorites',
6302         'only_matching': True,
6303     }]
6304
6305     def _real_extract(self, url):
6306         return self.url_result(
6307             'https://www.youtube.com/playlist?list=LL',
6308             ie=YoutubeTabIE.ie_key())
6309
6310
6311 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6312     IE_NAME = 'youtube:notif'
6313     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6314     _VALID_URL = r':ytnotif(?:ication)?s?'
6315     _LOGIN_REQUIRED = True
6316     _TESTS = [{
6317         'url': ':ytnotif',
6318         'only_matching': True,
6319     }, {
6320         'url': ':ytnotifications',
6321         'only_matching': True,
6322     }]
6323
6324     def _extract_notification_menu(self, response, continuation_list):
6325         notification_list = traverse_obj(
6326             response,
6327             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6328             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6329             expected_type=list) or []
6330         continuation_list[0] = None
6331         for item in notification_list:
6332             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6333             if entry:
6334                 yield entry
6335             continuation = item.get('continuationItemRenderer')
6336             if continuation:
6337                 continuation_list[0] = continuation
6338
6339     def _extract_notification_renderer(self, notification):
6340         video_id = traverse_obj(
6341             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6342         url = f'https://www.youtube.com/watch?v={video_id}'
6343         channel_id = None
6344         if not video_id:
6345             browse_ep = traverse_obj(
6346                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6347             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6348             post_id = self._search_regex(
6349                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6350                 'post id', default=None)
6351             if not channel_id or not post_id:
6352                 return
6353             # The direct /post url redirects to this in the browser
6354             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6355
6356         channel = traverse_obj(
6357             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6358             expected_type=str)
6359         notification_title = self._get_text(notification, 'shortMessage')
6360         if notification_title:
6361             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6362         # TODO: handle recommended videos
6363         title = self._search_regex(
6364             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6365             'video title', default=None)
6366         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6367                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6368                      else None)
6369         return {
6370             '_type': 'url',
6371             'url': url,
6372             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6373             'video_id': video_id,
6374             'title': title,
6375             'channel_id': channel_id,
6376             'channel': channel,
6377             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6378             'timestamp': timestamp,
6379         }
6380
6381     def _notification_menu_entries(self, ytcfg):
6382         continuation_list = [None]
6383         response = None
6384         for page in itertools.count(1):
6385             ctoken = traverse_obj(
6386                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6387             response = self._extract_response(
6388                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6389                 ep='notification/get_notification_menu', check_get_keys='actions',
6390                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6391             yield from self._extract_notification_menu(response, continuation_list)
6392             if not continuation_list[0]:
6393                 break
6394
6395     def _real_extract(self, url):
6396         display_id = 'notifications'
6397         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6398         self._report_playlist_authcheck(ytcfg)
6399         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6400
6401
6402 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6403     IE_DESC = 'YouTube search'
6404     IE_NAME = 'youtube:search'
6405     _SEARCH_KEY = 'ytsearch'
6406     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6407     _TESTS = [{
6408         'url': 'ytsearch5:youtube-dl test video',
6409         'playlist_count': 5,
6410         'info_dict': {
6411             'id': 'youtube-dl test video',
6412             'title': 'youtube-dl test video',
6413         }
6414     }]
6415
6416
6417 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6418     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6419     _SEARCH_KEY = 'ytsearchdate'
6420     IE_DESC = 'YouTube search, newest videos first'
6421     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6422     _TESTS = [{
6423         'url': 'ytsearchdate5:youtube-dl test video',
6424         'playlist_count': 5,
6425         'info_dict': {
6426             'id': 'youtube-dl test video',
6427             'title': 'youtube-dl test video',
6428         }
6429     }]
6430
6431
6432 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6433     IE_DESC = 'YouTube search URLs with sorting and filter support'
6434     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6435     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6436     _TESTS = [{
6437         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6438         'playlist_mincount': 5,
6439         'info_dict': {
6440             'id': 'youtube-dl test video',
6441             'title': 'youtube-dl test video',
6442         }
6443     }, {
6444         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6445         'playlist_mincount': 5,
6446         'info_dict': {
6447             'id': 'python',
6448             'title': 'python',
6449         }
6450     }, {
6451         'url': 'https://www.youtube.com/results?search_query=%23cats',
6452         'playlist_mincount': 1,
6453         'info_dict': {
6454             'id': '#cats',
6455             'title': '#cats',
6456             # The test suite does not have support for nested playlists
6457             # 'entries': [{
6458             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6459             #     'title': '#cats',
6460             # }],
6461         },
6462     }, {
6463         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6464         'only_matching': True,
6465     }]
6466
6467     def _real_extract(self, url):
6468         qs = parse_qs(url)
6469         query = (qs.get('search_query') or qs.get('q'))[0]
6470         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6471
6472
6473 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6474     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6475     IE_NAME = 'youtube:music:search_url'
6476     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6477     _TESTS = [{
6478         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6479         'playlist_count': 16,
6480         'info_dict': {
6481             'id': 'royalty free music',
6482             'title': 'royalty free music',
6483         }
6484     }, {
6485         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6486         'playlist_mincount': 30,
6487         'info_dict': {
6488             'id': 'royalty free music - songs',
6489             'title': 'royalty free music - songs',
6490         },
6491         'params': {'extract_flat': 'in_playlist'}
6492     }, {
6493         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6494         'playlist_mincount': 30,
6495         'info_dict': {
6496             'id': 'royalty free music - community playlists',
6497             'title': 'royalty free music - community playlists',
6498         },
6499         'params': {'extract_flat': 'in_playlist'}
6500     }]
6501
6502     _SECTIONS = {
6503         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6504         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6505         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6506         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6507         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6508         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6509     }
6510
6511     def _real_extract(self, url):
6512         qs = parse_qs(url)
6513         query = (qs.get('search_query') or qs.get('q'))[0]
6514         params = qs.get('sp', (None,))[0]
6515         if params:
6516             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6517         else:
6518             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6519             params = self._SECTIONS.get(section)
6520             if not params:
6521                 section = None
6522         title = join_nonempty(query, section, delim=' - ')
6523         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6524
6525
6526 class YoutubeFeedsInfoExtractor(InfoExtractor):
6527     """
6528     Base class for feed extractors
6529     Subclasses must re-define the _FEED_NAME property.
6530     """
6531     _LOGIN_REQUIRED = True
6532     _FEED_NAME = 'feeds'
6533
6534     def _real_initialize(self):
6535         YoutubeBaseInfoExtractor._check_login_required(self)
6536
6537     @classproperty
6538     def IE_NAME(self):
6539         return f'youtube:{self._FEED_NAME}'
6540
6541     def _real_extract(self, url):
6542         return self.url_result(
6543             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6544
6545
6546 class YoutubeWatchLaterIE(InfoExtractor):
6547     IE_NAME = 'youtube:watchlater'
6548     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6549     _VALID_URL = r':ytwatchlater'
6550     _TESTS = [{
6551         'url': ':ytwatchlater',
6552         'only_matching': True,
6553     }]
6554
6555     def _real_extract(self, url):
6556         return self.url_result(
6557             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6558
6559
6560 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6561     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6562     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6563     _FEED_NAME = 'recommended'
6564     _LOGIN_REQUIRED = False
6565     _TESTS = [{
6566         'url': ':ytrec',
6567         'only_matching': True,
6568     }, {
6569         'url': ':ytrecommended',
6570         'only_matching': True,
6571     }, {
6572         'url': 'https://youtube.com',
6573         'only_matching': True,
6574     }]
6575
6576
6577 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6578     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6579     _VALID_URL = r':ytsub(?:scription)?s?'
6580     _FEED_NAME = 'subscriptions'
6581     _TESTS = [{
6582         'url': ':ytsubs',
6583         'only_matching': True,
6584     }, {
6585         'url': ':ytsubscriptions',
6586         'only_matching': True,
6587     }]
6588
6589
6590 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6591     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6592     _VALID_URL = r':ythis(?:tory)?'
6593     _FEED_NAME = 'history'
6594     _TESTS = [{
6595         'url': ':ythistory',
6596         'only_matching': True,
6597     }]
6598
6599
6600 class YoutubeStoriesIE(InfoExtractor):
6601     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6602     IE_NAME = 'youtube:stories'
6603     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6604     _TESTS = [{
6605         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6606         'only_matching': True,
6607     }]
6608
6609     def _real_extract(self, url):
6610         playlist_id = f'RLTD{self._match_id(url)}'
6611         return self.url_result(
6612             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6613             ie=YoutubeTabIE, video_id=playlist_id)
6614
6615
6616 class YoutubeShortsAudioPivotIE(InfoExtractor):
6617     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6618     IE_NAME = 'youtube:shorts:pivot:audio'
6619     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6620     _TESTS = [{
6621         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6622         'only_matching': True,
6623     }]
6624
6625     @staticmethod
6626     def _generate_audio_pivot_params(video_id):
6627         """
6628         Generates sfv_audio_pivot browse params for this video id
6629         """
6630         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6631         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6632
6633     def _real_extract(self, url):
6634         video_id = self._match_id(url)
6635         return self.url_result(
6636             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6637             ie=YoutubeTabIE)
6638
6639
6640 class YoutubeTruncatedURLIE(InfoExtractor):
6641     IE_NAME = 'youtube:truncated_url'
6642     IE_DESC = False  # Do not list
6643     _VALID_URL = r'''(?x)
6644         (?:https?://)?
6645         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6646         (?:watch\?(?:
6647             feature=[a-z_]+|
6648             annotation_id=annotation_[^&]+|
6649             x-yt-cl=[0-9]+|
6650             hl=[^&]*|
6651             t=[0-9]+
6652         )?
6653         |
6654             attribution_link\?a=[^&]+
6655         )
6656         $
6657     '''
6658
6659     _TESTS = [{
6660         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6661         'only_matching': True,
6662     }, {
6663         'url': 'https://www.youtube.com/watch?',
6664         'only_matching': True,
6665     }, {
6666         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6667         'only_matching': True,
6668     }, {
6669         'url': 'https://www.youtube.com/watch?feature=foo',
6670         'only_matching': True,
6671     }, {
6672         'url': 'https://www.youtube.com/watch?hl=en-GB',
6673         'only_matching': True,
6674     }, {
6675         'url': 'https://www.youtube.com/watch?t=2372',
6676         'only_matching': True,
6677     }]
6678
6679     def _real_extract(self, url):
6680         raise ExtractorError(
6681             'Did you forget to quote the URL? Remember that & is a meta '
6682             'character in most shells, so you want to put the URL in quotes, '
6683             'like  youtube-dl '
6684             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6685             ' or simply  youtube-dl BaW_jenozKc  .',
6686             expected=True)
6687
6688
6689 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6690     IE_NAME = 'youtube:clip'
6691     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6692     _TESTS = [{
6693         # FIXME: Other metadata should be extracted from the clip, not from the base video
6694         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6695         'info_dict': {
6696             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6697             'ext': 'mp4',
6698             'section_start': 29.0,
6699             'section_end': 39.7,
6700             'duration': 10.7,
6701             'age_limit': 0,
6702             'availability': 'public',
6703             'categories': ['Gaming'],
6704             'channel': 'Scott The Woz',
6705             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6706             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6707             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6708             'like_count': int,
6709             'playable_in_embed': True,
6710             'tags': 'count:17',
6711             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6712             'title': 'Mobile Games on Console - Scott The Woz',
6713             'upload_date': '20210920',
6714             'uploader': 'Scott The Woz',
6715             'uploader_id': 'scottthewoz',
6716             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6717             'view_count': int,
6718             'live_status': 'not_live',
6719             'channel_follower_count': int
6720         }
6721     }]
6722
6723     def _real_extract(self, url):
6724         clip_id = self._match_id(url)
6725         _, data = self._extract_webpage(url, clip_id)
6726
6727         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6728         if not video_id:
6729             raise ExtractorError('Unable to find video ID')
6730
6731         clip_data = traverse_obj(data, (
6732             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6733             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6734             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6735             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6736
6737         return {
6738             '_type': 'url_transparent',
6739             'url': f'https://www.youtube.com/watch?v={video_id}',
6740             'ie_key': YoutubeIE.ie_key(),
6741             'id': clip_id,
6742             'section_start': int(clip_data['startTimeMs']) / 1000,
6743             'section_end': int(clip_data['endTimeMs']) / 1000,
6744         }
6745
6746
6747 class YoutubeTruncatedIDIE(InfoExtractor):
6748     IE_NAME = 'youtube:truncated_id'
6749     IE_DESC = False  # Do not list
6750     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6751
6752     _TESTS = [{
6753         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6754         'only_matching': True,
6755     }]
6756
6757     def _real_extract(self, url):
6758         video_id = self._match_id(url)
6759         raise ExtractorError(
6760             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6761             expected=True)