yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import enum
   6 import hashlib
   7 import itertools
   8 import json
   9 import math
  10 import os.path
  11 import random
  12 import re
  13 import sys
  14 import threading
  15 import time
  16 import traceback
  17 import urllib.error
  18 import urllib.parse
  19
  20 from .common import InfoExtractor, SearchInfoExtractor
  21 from .openload import PhantomJSwrapper
  22 from ..compat import functools
  23 from ..jsinterp import JSInterpreter
  24 from ..utils import (
  25     NO_DEFAULT,
  26     ExtractorError,
  27     LazyList,
  28     UserNotLive,
  29     bug_reports_message,
  30     classproperty,
  31     clean_html,
  32     datetime_from_str,
  33     dict_get,
  34     filter_dict,
  35     float_or_none,
  36     format_field,
  37     get_first,
  38     int_or_none,
  39     is_html,
  40     join_nonempty,
  41     js_to_json,
  42     mimetype2ext,
  43     network_exceptions,
  44     orderedSet,
  45     parse_codecs,
  46     parse_count,
  47     parse_duration,
  48     parse_iso8601,
  49     parse_qs,
  50     qualities,
  51     remove_start,
  52     smuggle_url,
  53     str_or_none,
  54     str_to_int,
  55     strftime_or_none,
  56     traverse_obj,
  57     try_get,
  58     unescapeHTML,
  59     unified_strdate,
  60     unified_timestamp,
  61     unsmuggle_url,
  62     update_url_query,
  63     url_or_none,
  64     urljoin,
  65     variadic,
  66 )
  67
  68 # any clients starting with _ cannot be explicitly requested by the user
  69 INNERTUBE_CLIENTS = {
  70     'web': {
  71         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  72         'INNERTUBE_CONTEXT': {
  73             'client': {
  74                 'clientName': 'WEB',
  75                 'clientVersion': '2.20220801.00.00',
  76             }
  77         },
  78         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  79     },
  80     'web_embedded': {
  81         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  82         'INNERTUBE_CONTEXT': {
  83             'client': {
  84                 'clientName': 'WEB_EMBEDDED_PLAYER',
  85                 'clientVersion': '1.20220731.00.00',
  86             },
  87         },
  88         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  89     },
  90     'web_music': {
  91         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  92         'INNERTUBE_HOST': 'music.youtube.com',
  93         'INNERTUBE_CONTEXT': {
  94             'client': {
  95                 'clientName': 'WEB_REMIX',
  96                 'clientVersion': '1.20220727.01.00',
  97             }
  98         },
  99         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 100     },
 101     'web_creator': {
 102         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 103         'INNERTUBE_CONTEXT': {
 104             'client': {
 105                 'clientName': 'WEB_CREATOR',
 106                 'clientVersion': '1.20220726.00.00',
 107             }
 108         },
 109         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 110     },
 111     'android': {
 112         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 113         'INNERTUBE_CONTEXT': {
 114             'client': {
 115                 'clientName': 'ANDROID',
 116                 'clientVersion': '17.31.35',
 117                 'androidSdkVersion': 30,
 118                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 119             }
 120         },
 121         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 122         'REQUIRE_JS_PLAYER': False
 123     },
 124     'android_embedded': {
 125         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 126         'INNERTUBE_CONTEXT': {
 127             'client': {
 128                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 129                 'clientVersion': '17.31.35',
 130                 'androidSdkVersion': 30,
 131                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 132             },
 133         },
 134         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 135         'REQUIRE_JS_PLAYER': False
 136     },
 137     'android_music': {
 138         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 139         'INNERTUBE_CONTEXT': {
 140             'client': {
 141                 'clientName': 'ANDROID_MUSIC',
 142                 'clientVersion': '5.16.51',
 143                 'androidSdkVersion': 30,
 144                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 145             }
 146         },
 147         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 148         'REQUIRE_JS_PLAYER': False
 149     },
 150     'android_creator': {
 151         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 152         'INNERTUBE_CONTEXT': {
 153             'client': {
 154                 'clientName': 'ANDROID_CREATOR',
 155                 'clientVersion': '22.30.100',
 156                 'androidSdkVersion': 30,
 157                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 158             },
 159         },
 160         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 161         'REQUIRE_JS_PLAYER': False
 162     },
 163     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 164     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 165     'ios': {
 166         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 167         'INNERTUBE_CONTEXT': {
 168             'client': {
 169                 'clientName': 'IOS',
 170                 'clientVersion': '17.33.2',
 171                 'deviceModel': 'iPhone14,3',
 172                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 173             }
 174         },
 175         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 176         'REQUIRE_JS_PLAYER': False
 177     },
 178     'ios_embedded': {
 179         'INNERTUBE_CONTEXT': {
 180             'client': {
 181                 'clientName': 'IOS_MESSAGES_EXTENSION',
 182                 'clientVersion': '17.33.2',
 183                 'deviceModel': 'iPhone14,3',
 184                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 185             },
 186         },
 187         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 188         'REQUIRE_JS_PLAYER': False
 189     },
 190     'ios_music': {
 191         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 192         'INNERTUBE_CONTEXT': {
 193             'client': {
 194                 'clientName': 'IOS_MUSIC',
 195                 'clientVersion': '5.21',
 196                 'deviceModel': 'iPhone14,3',
 197                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 198             },
 199         },
 200         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 201         'REQUIRE_JS_PLAYER': False
 202     },
 203     'ios_creator': {
 204         'INNERTUBE_CONTEXT': {
 205             'client': {
 206                 'clientName': 'IOS_CREATOR',
 207                 'clientVersion': '22.33.101',
 208                 'deviceModel': 'iPhone14,3',
 209                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 210             },
 211         },
 212         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 213         'REQUIRE_JS_PLAYER': False
 214     },
 215     # mweb has 'ultralow' formats
 216     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 217     'mweb': {
 218         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 219         'INNERTUBE_CONTEXT': {
 220             'client': {
 221                 'clientName': 'MWEB',
 222                 'clientVersion': '2.20220801.00.00',
 223             }
 224         },
 225         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 226     },
 227     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 228     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 229     'tv_embedded': {
 230         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 231         'INNERTUBE_CONTEXT': {
 232             'client': {
 233                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 234                 'clientVersion': '2.0',
 235             },
 236         },
 237         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 238     },
 239 }
 240
 241
 242 def _split_innertube_client(client_name):
 243     variant, *base = client_name.rsplit('.', 1)
 244     if base:
 245         return variant, base[0], variant
 246     base, *variant = client_name.split('_', 1)
 247     return client_name, base, variant[0] if variant else None
 248
 249
 250 def build_innertube_clients():
 251     THIRD_PARTY = {
 252         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 253     }
 254     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 255     priority = qualities(BASE_CLIENTS[::-1])
 256
 257     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 258         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 259         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 260         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 261         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 262
 263         _, base_client, variant = _split_innertube_client(client)
 264         ytcfg['priority'] = 10 * priority(base_client)
 265
 266         if not variant:
 267             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 268             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 269             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 270             embedscreen['priority'] -= 3
 271         elif variant == 'embedded':
 272             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 273             ytcfg['priority'] -= 2
 274         else:
 275             ytcfg['priority'] -= 3
 276
 277
 278 build_innertube_clients()
 279
 280
 281 class BadgeType(enum.Enum):
 282     AVAILABILITY_UNLISTED = enum.auto()
 283     AVAILABILITY_PRIVATE = enum.auto()
 284     AVAILABILITY_PUBLIC = enum.auto()
 285     AVAILABILITY_PREMIUM = enum.auto()
 286     AVAILABILITY_SUBSCRIPTION = enum.auto()
 287     LIVE_NOW = enum.auto()
 288
 289
 290 class YoutubeBaseInfoExtractor(InfoExtractor):
 291     """Provide base functions for Youtube extractors"""
 292
 293     _RESERVED_NAMES = (
 294         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 295         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 296         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 297         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 298
 299     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 300
 301     # _NETRC_MACHINE = 'youtube'
 302
 303     # If True it will raise an error if no login info is provided
 304     _LOGIN_REQUIRED = False
 305
 306     _INVIDIOUS_SITES = (
 307         # invidious-redirect websites
 308         r'(?:www\.)?redirect\.invidious\.io',
 309         r'(?:(?:www|dev)\.)?invidio\.us',
 310         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 311         r'(?:www\.)?invidious\.pussthecat\.org',
 312         r'(?:www\.)?invidious\.zee\.li',
 313         r'(?:www\.)?invidious\.ethibox\.fr',
 314         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 315         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 316         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 317         # youtube-dl invidious instances list
 318         r'(?:(?:www|no)\.)?invidiou\.sh',
 319         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 320         r'(?:www\.)?invidious\.kabi\.tk',
 321         r'(?:www\.)?invidious\.mastodon\.host',
 322         r'(?:www\.)?invidious\.zapashcanon\.fr',
 323         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 324         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 325         r'(?:www\.)?invidious\.himiko\.cloud',
 326         r'(?:www\.)?invidious\.reallyancient\.tech',
 327         r'(?:www\.)?invidious\.tube',
 328         r'(?:www\.)?invidiou\.site',
 329         r'(?:www\.)?invidious\.site',
 330         r'(?:www\.)?invidious\.xyz',
 331         r'(?:www\.)?invidious\.nixnet\.xyz',
 332         r'(?:www\.)?invidious\.048596\.xyz',
 333         r'(?:www\.)?invidious\.drycat\.fr',
 334         r'(?:www\.)?inv\.skyn3t\.in',
 335         r'(?:www\.)?tube\.poal\.co',
 336         r'(?:www\.)?tube\.connect\.cafe',
 337         r'(?:www\.)?vid\.wxzm\.sx',
 338         r'(?:www\.)?vid\.mint\.lgbt',
 339         r'(?:www\.)?vid\.puffyan\.us',
 340         r'(?:www\.)?yewtu\.be',
 341         r'(?:www\.)?yt\.elukerio\.org',
 342         r'(?:www\.)?yt\.lelux\.fi',
 343         r'(?:www\.)?invidious\.ggc-project\.de',
 344         r'(?:www\.)?yt\.maisputain\.ovh',
 345         r'(?:www\.)?ytprivate\.com',
 346         r'(?:www\.)?invidious\.13ad\.de',
 347         r'(?:www\.)?invidious\.toot\.koeln',
 348         r'(?:www\.)?invidious\.fdn\.fr',
 349         r'(?:www\.)?watch\.nettohikari\.com',
 350         r'(?:www\.)?invidious\.namazso\.eu',
 351         r'(?:www\.)?invidious\.silkky\.cloud',
 352         r'(?:www\.)?invidious\.exonip\.de',
 353         r'(?:www\.)?invidious\.riverside\.rocks',
 354         r'(?:www\.)?invidious\.blamefran\.net',
 355         r'(?:www\.)?invidious\.moomoo\.de',
 356         r'(?:www\.)?ytb\.trom\.tf',
 357         r'(?:www\.)?yt\.cyberhost\.uk',
 358         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 359         r'(?:www\.)?qklhadlycap4cnod\.onion',
 360         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 361         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 362         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 363         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 364         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 365         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 366         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 367         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 368         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 369         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 370         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 371         r'(?:www\.)?piped\.kavin\.rocks',
 372         r'(?:www\.)?piped\.tokhmi\.xyz',
 373         r'(?:www\.)?piped\.syncpundit\.io',
 374         r'(?:www\.)?piped\.mha\.fi',
 375         r'(?:www\.)?watch\.whatever\.social',
 376         r'(?:www\.)?piped\.garudalinux\.org',
 377         r'(?:www\.)?piped\.rivo\.lol',
 378         r'(?:www\.)?piped-libre\.kavin\.rocks',
 379         r'(?:www\.)?yt\.jae\.fi',
 380         r'(?:www\.)?piped\.mint\.lgbt',
 381         r'(?:www\.)?il\.ax',
 382         r'(?:www\.)?piped\.esmailelbob\.xyz',
 383         r'(?:www\.)?piped\.projectsegfau\.lt',
 384         r'(?:www\.)?piped\.privacydev\.net',
 385         r'(?:www\.)?piped\.palveluntarjoaja\.eu',
 386         r'(?:www\.)?piped\.smnz\.de',
 387         r'(?:www\.)?piped\.adminforge\.de',
 388         r'(?:www\.)?watch\.whatevertinfoil\.de',
 389         r'(?:www\.)?piped\.qdi\.fi',
 390     )
 391
 392     # extracted from account/account_menu ep
 393     # XXX: These are the supported YouTube UI and API languages,
 394     # which is slightly different from languages supported for translation in YouTube studio
 395     _SUPPORTED_LANG_CODES = [
 396         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 397         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 398         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 399         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 400         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 401         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 402     ]
 403
 404     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 405
 406     @functools.cached_property
 407     def _preferred_lang(self):
 408         """
 409         Returns a language code supported by YouTube for the user preferred language.
 410         Returns None if no preferred language set.
 411         """
 412         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 413         if not preferred_lang:
 414             return
 415         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 416             raise ExtractorError(
 417                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 418                 expected=True)
 419         elif preferred_lang != 'en':
 420             self.report_warning(
 421                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 422         return preferred_lang
 423
 424     def _initialize_consent(self):
 425         cookies = self._get_cookies('https://www.youtube.com/')
 426         if cookies.get('__Secure-3PSID'):
 427             return
 428         consent_id = None
 429         consent = cookies.get('CONSENT')
 430         if consent:
 431             if 'YES' in consent.value:
 432                 return
 433             consent_id = self._search_regex(
 434                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 435         if not consent_id:
 436             consent_id = random.randint(100, 999)
 437         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 438
 439     def _initialize_pref(self):
 440         cookies = self._get_cookies('https://www.youtube.com/')
 441         pref_cookie = cookies.get('PREF')
 442         pref = {}
 443         if pref_cookie:
 444             try:
 445                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 446             except ValueError:
 447                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 448         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 449         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 450
 451     def _real_initialize(self):
 452         self._initialize_pref()
 453         self._initialize_consent()
 454         self._check_login_required()
 455
 456     def _check_login_required(self):
 457         if self._LOGIN_REQUIRED and not self._cookies_passed:
 458             self.raise_login_required('Login details are needed to download this content', method='cookies')
 459
 460     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 461     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 462
 463     def _get_default_ytcfg(self, client='web'):
 464         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 465
 466     def _get_innertube_host(self, client='web'):
 467         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 468
 469     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 470         # try_get but with fallback to default ytcfg client values when present
 471         _func = lambda y: try_get(y, getter, expected_type)
 472         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 473
 474     def _extract_client_name(self, ytcfg, default_client='web'):
 475         return self._ytcfg_get_safe(
 476             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 477                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 478
 479     def _extract_client_version(self, ytcfg, default_client='web'):
 480         return self._ytcfg_get_safe(
 481             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 482                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 483
 484     def _select_api_hostname(self, req_api_hostname, default_client=None):
 485         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 486                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 487
 488     def _extract_api_key(self, ytcfg=None, default_client='web'):
 489         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 490
 491     def _extract_context(self, ytcfg=None, default_client='web'):
 492         context = get_first(
 493             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 494         # Enforce language and tz for extraction
 495         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 496         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 497         return context
 498
 499     _SAPISID = None
 500
 501     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 502         time_now = round(time.time())
 503         if self._SAPISID is None:
 504             yt_cookies = self._get_cookies('https://www.youtube.com')
 505             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 506             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 507             sapisid_cookie = dict_get(
 508                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 509             if sapisid_cookie and sapisid_cookie.value:
 510                 self._SAPISID = sapisid_cookie.value
 511                 self.write_debug('Extracted SAPISID cookie')
 512                 # SAPISID cookie is required if not already present
 513                 if not yt_cookies.get('SAPISID'):
 514                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 515                     self._set_cookie(
 516                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 517             else:
 518                 self._SAPISID = False
 519         if not self._SAPISID:
 520             return None
 521         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 522         sapisidhash = hashlib.sha1(
 523             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 524         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 525
 526     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 527                   note='Downloading API JSON', errnote='Unable to download API page',
 528                   context=None, api_key=None, api_hostname=None, default_client='web'):
 529
 530         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 531         data.update(query)
 532         real_headers = self.generate_api_headers(default_client=default_client)
 533         real_headers.update({'content-type': 'application/json'})
 534         if headers:
 535             real_headers.update(headers)
 536         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 537                    or api_key or self._extract_api_key(default_client=default_client))
 538         return self._download_json(
 539             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 540             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 541             data=json.dumps(data).encode('utf8'), headers=real_headers,
 542             query={'key': api_key, 'prettyPrint': 'false'})
 543
 544     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 545         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 546
 547     @staticmethod
 548     def _extract_session_index(*data):
 549         """
 550         Index of current account in account list.
 551         See: https://github.com/yt-dlp/yt-dlp/pull/519
 552         """
 553         for ytcfg in data:
 554             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 555             if session_index is not None:
 556                 return session_index
 557
 558     # Deprecated?
 559     def _extract_identity_token(self, ytcfg=None, webpage=None):
 560         if ytcfg:
 561             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 562             if token:
 563                 return token
 564         if webpage:
 565             return self._search_regex(
 566                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 567                 'identity token', default=None, fatal=False)
 568
 569     @staticmethod
 570     def _extract_account_syncid(*args):
 571         """
 572         Extract syncId required to download private playlists of secondary channels
 573         @params response and/or ytcfg
 574         """
 575         for data in args:
 576             # ytcfg includes channel_syncid if on secondary channel
 577             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 578             if delegated_sid:
 579                 return delegated_sid
 580             sync_ids = (try_get(
 581                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 582                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 583             if len(sync_ids) >= 2 and sync_ids[1]:
 584                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 585                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 586                 return sync_ids[0]
 587
 588     @staticmethod
 589     def _extract_visitor_data(*args):
 590         """
 591         Extracts visitorData from an API response or ytcfg
 592         Appears to be used to track session state
 593         """
 594         return get_first(
 595             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 596             expected_type=str)
 597
 598     @functools.cached_property
 599     def is_authenticated(self):
 600         return bool(self._generate_sapisidhash_header())
 601
 602     def extract_ytcfg(self, video_id, webpage):
 603         if not webpage:
 604             return {}
 605         return self._parse_json(
 606             self._search_regex(
 607                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 608                 default='{}'), video_id, fatal=False) or {}
 609
 610     def generate_api_headers(
 611             self, *, ytcfg=None, account_syncid=None, session_index=None,
 612             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 613
 614         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 615         headers = {
 616             'X-YouTube-Client-Name': str(
 617                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 618             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 619             'Origin': origin,
 620             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 621             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 622             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 623             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 624         }
 625         if session_index is None:
 626             session_index = self._extract_session_index(ytcfg)
 627         if account_syncid or session_index is not None:
 628             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 629
 630         auth = self._generate_sapisidhash_header(origin)
 631         if auth is not None:
 632             headers['Authorization'] = auth
 633             headers['X-Origin'] = origin
 634         return filter_dict(headers)
 635
 636     def _download_ytcfg(self, client, video_id):
 637         url = {
 638             'web': 'https://www.youtube.com',
 639             'web_music': 'https://music.youtube.com',
 640             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 641         }.get(client)
 642         if not url:
 643             return {}
 644         webpage = self._download_webpage(
 645             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 646         return self.extract_ytcfg(video_id, webpage) or {}
 647
 648     @staticmethod
 649     def _build_api_continuation_query(continuation, ctp=None):
 650         query = {
 651             'continuation': continuation
 652         }
 653         # TODO: Inconsistency with clickTrackingParams.
 654         # Currently we have a fixed ctp contained within context (from ytcfg)
 655         # and a ctp in root query for continuation.
 656         if ctp:
 657             query['clickTracking'] = {'clickTrackingParams': ctp}
 658         return query
 659
 660     @classmethod
 661     def _extract_next_continuation_data(cls, renderer):
 662         next_continuation = try_get(
 663             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 664                        lambda x: x['continuation']['reloadContinuationData']), dict)
 665         if not next_continuation:
 666             return
 667         continuation = next_continuation.get('continuation')
 668         if not continuation:
 669             return
 670         ctp = next_continuation.get('clickTrackingParams')
 671         return cls._build_api_continuation_query(continuation, ctp)
 672
 673     @classmethod
 674     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 675         if isinstance(continuation_ep, dict):
 676             continuation = try_get(
 677                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 678             if not continuation:
 679                 return
 680             ctp = continuation_ep.get('clickTrackingParams')
 681             return cls._build_api_continuation_query(continuation, ctp)
 682
 683     @classmethod
 684     def _extract_continuation(cls, renderer):
 685         next_continuation = cls._extract_next_continuation_data(renderer)
 686         if next_continuation:
 687             return next_continuation
 688
 689         return traverse_obj(renderer, (
 690             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 691             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 692         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 693
 694     @classmethod
 695     def _extract_alerts(cls, data):
 696         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 697             if not isinstance(alert_dict, dict):
 698                 continue
 699             for alert in alert_dict.values():
 700                 alert_type = alert.get('type')
 701                 if not alert_type:
 702                     continue
 703                 message = cls._get_text(alert, 'text')
 704                 if message:
 705                     yield alert_type, message
 706
 707     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 708         errors, warnings = [], []
 709         for alert_type, alert_message in alerts:
 710             if alert_type.lower() == 'error' and fatal:
 711                 errors.append([alert_type, alert_message])
 712             elif alert_message not in self._IGNORED_WARNINGS:
 713                 warnings.append([alert_type, alert_message])
 714
 715         for alert_type, alert_message in (warnings + errors[:-1]):
 716             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 717         if errors:
 718             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 719
 720     def _extract_and_report_alerts(self, data, *args, **kwargs):
 721         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 722
 723     def _extract_badges(self, renderer: dict):
 724         privacy_icon_map = {
 725             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 726             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 727             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 728         }
 729
 730         badge_style_map = {
 731             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 732             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 733             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 734         }
 735
 736         label_map = {
 737             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 738             'private': BadgeType.AVAILABILITY_PRIVATE,
 739             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 740             'live': BadgeType.LIVE_NOW,
 741             'premium': BadgeType.AVAILABILITY_PREMIUM
 742         }
 743
 744         badges = []
 745         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer'), default=[]):
 746             badge_type = (
 747                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 748                 or badge_style_map.get(traverse_obj(badge, 'style'))
 749             )
 750             if badge_type:
 751                 badges.append({'type': badge_type})
 752                 continue
 753
 754             # fallback, won't work in some languages
 755             label = traverse_obj(badge, 'label', expected_type=str, default='')
 756             for match, label_badge_type in label_map.items():
 757                 if match in label.lower():
 758                     badges.append({'type': badge_type})
 759                     continue
 760
 761         return badges
 762
 763     @staticmethod
 764     def _has_badge(badges, badge_type):
 765         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 766
 767     @staticmethod
 768     def _get_text(data, *path_list, max_runs=None):
 769         for path in path_list or [None]:
 770             if path is None:
 771                 obj = [data]
 772             else:
 773                 obj = traverse_obj(data, path, default=[])
 774                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 775                     obj = [obj]
 776             for item in obj:
 777                 text = try_get(item, lambda x: x['simpleText'], str)
 778                 if text:
 779                     return text
 780                 runs = try_get(item, lambda x: x['runs'], list) or []
 781                 if not runs and isinstance(item, list):
 782                     runs = item
 783
 784                 runs = runs[:min(len(runs), max_runs or len(runs))]
 785                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 786                 if text:
 787                     return text
 788
 789     def _get_count(self, data, *path_list):
 790         count_text = self._get_text(data, *path_list) or ''
 791         count = parse_count(count_text)
 792         if count is None:
 793             count = str_to_int(
 794                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 795         return count
 796
 797     @staticmethod
 798     def _extract_thumbnails(data, *path_list):
 799         """
 800         Extract thumbnails from thumbnails dict
 801         @param path_list: path list to level that contains 'thumbnails' key
 802         """
 803         thumbnails = []
 804         for path in path_list or [()]:
 805             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 806                 thumbnail_url = url_or_none(thumbnail.get('url'))
 807                 if not thumbnail_url:
 808                     continue
 809                 # Sometimes youtube gives a wrong thumbnail URL. See:
 810                 # https://github.com/yt-dlp/yt-dlp/issues/233
 811                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 812                 if 'maxresdefault' in thumbnail_url:
 813                     thumbnail_url = thumbnail_url.split('?')[0]
 814                 thumbnails.append({
 815                     'url': thumbnail_url,
 816                     'height': int_or_none(thumbnail.get('height')),
 817                     'width': int_or_none(thumbnail.get('width')),
 818                 })
 819         return thumbnails
 820
 821     @staticmethod
 822     def extract_relative_time(relative_time_text):
 823         """
 824         Extracts a relative time from string and converts to dt object
 825         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 826         """
 827         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 828         if mobj:
 829             start = mobj.group('start')
 830             if start:
 831                 return datetime_from_str(start)
 832             try:
 833                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 834             except ValueError:
 835                 return None
 836
 837     def _parse_time_text(self, text):
 838         if not text:
 839             return
 840         dt = self.extract_relative_time(text)
 841         timestamp = None
 842         if isinstance(dt, datetime.datetime):
 843             timestamp = calendar.timegm(dt.timetuple())
 844
 845         if timestamp is None:
 846             timestamp = (
 847                 unified_timestamp(text) or unified_timestamp(
 848                     self._search_regex(
 849                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 850                         text.lower(), 'time text', default=None)))
 851
 852         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 853             self.report_warning(
 854                 f'Cannot parse localized time text "{text}"', only_once=True)
 855         return timestamp
 856
 857     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 858                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 859                           default_client='web'):
 860         for retry in self.RetryManager():
 861             try:
 862                 response = self._call_api(
 863                     ep=ep, fatal=True, headers=headers,
 864                     video_id=item_id, query=query, note=note,
 865                     context=self._extract_context(ytcfg, default_client),
 866                     api_key=self._extract_api_key(ytcfg, default_client),
 867                     api_hostname=api_hostname, default_client=default_client)
 868             except ExtractorError as e:
 869                 if not isinstance(e.cause, network_exceptions):
 870                     return self._error_or_warning(e, fatal=fatal)
 871                 elif not isinstance(e.cause, urllib.error.HTTPError):
 872                     retry.error = e
 873                     continue
 874
 875                 first_bytes = e.cause.read(512)
 876                 if not is_html(first_bytes):
 877                     yt_error = try_get(
 878                         self._parse_json(
 879                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 880                         lambda x: x['error']['message'], str)
 881                     if yt_error:
 882                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 883                 # Downloading page may result in intermittent 5xx HTTP error
 884                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 885                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 886                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 887                 if e.cause.code not in (403, 429):
 888                     retry.error = e
 889                     continue
 890                 return self._error_or_warning(e, fatal=fatal)
 891
 892             try:
 893                 self._extract_and_report_alerts(response, only_once=True)
 894             except ExtractorError as e:
 895                 # YouTube servers may return errors we want to retry on in a 200 OK response
 896                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 897                 if 'unknown error' in e.msg.lower():
 898                     retry.error = e
 899                     continue
 900                 return self._error_or_warning(e, fatal=fatal)
 901             # Youtube sometimes sends incomplete data
 902             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 903             if not traverse_obj(response, *variadic(check_get_keys)):
 904                 retry.error = ExtractorError('Incomplete data received', expected=True)
 905                 continue
 906
 907             return response
 908
 909     @staticmethod
 910     def is_music_url(url):
 911         return re.match(r'https?://music\.youtube\.com/', url) is not None
 912
 913     def _extract_video(self, renderer):
 914         video_id = renderer.get('videoId')
 915         title = self._get_text(renderer, 'title')
 916         description = self._get_text(renderer, 'descriptionSnippet')
 917
 918         duration = int_or_none(renderer.get('lengthSeconds'))
 919         if duration is None:
 920             duration = parse_duration(self._get_text(
 921                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 922         if duration is None:
 923             duration = parse_duration(self._search_regex(
 924                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 925                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 926                 video_id, default=None, group='duration'))
 927
 928         # videoInfo is a string like '50K views • 10 years ago'.
 929         view_count = self._get_count(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo')
 930         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 931         channel_id = traverse_obj(
 932             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 933             expected_type=str, get_all=False)
 934         time_text = self._get_text(renderer, 'publishedTimeText', 'videoInfo') or ''
 935         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 936         overlay_style = traverse_obj(
 937             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 938             get_all=False, expected_type=str)
 939         badges = self._extract_badges(renderer)
 940         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 941         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 942             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 943             expected_type=str)) or ''
 944         url = f'https://www.youtube.com/watch?v={video_id}'
 945         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 946             url = f'https://www.youtube.com/shorts/{video_id}'
 947
 948         live_status = (
 949             'is_upcoming' if scheduled_timestamp is not None
 950             else 'was_live' if 'streamed' in time_text.lower()
 951             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
 952             else None)
 953
 954         return {
 955             '_type': 'url',
 956             'ie_key': YoutubeIE.ie_key(),
 957             'id': video_id,
 958             'url': url,
 959             'title': title,
 960             'description': description,
 961             'duration': duration,
 962             'uploader': uploader,
 963             'channel_id': channel_id,
 964             'thumbnails': thumbnails,
 965             'timestamp': (self._parse_time_text(time_text)
 966                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
 967                           else None),
 968             'release_timestamp': scheduled_timestamp,
 969             'availability':
 970                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
 971                 else self._availability(
 972                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
 973                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
 974                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
 975                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
 976             'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count': view_count,
 977             'live_status': live_status
 978         }
 979
 980
 981 class YoutubeIE(YoutubeBaseInfoExtractor):
 982     IE_DESC = 'YouTube'
 983     _VALID_URL = r"""(?x)^
 984                      (
 985                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 986                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 987                             (?:www\.)?deturl\.com/www\.youtube\.com|
 988                             (?:www\.)?pwnyoutube\.com|
 989                             (?:www\.)?hooktube\.com|
 990                             (?:www\.)?yourepeat\.com|
 991                             tube\.majestyc\.net|
 992                             %(invidious)s|
 993                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 994                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 995                          (?:                                                  # the various things that can precede the ID:
 996                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 997                              |(?:                                             # or the v= param in all its forms
 998                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 999                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
1000                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
1001                                  v=
1002                              )
1003                          ))
1004                          |(?:
1005                             youtu\.be|                                        # just youtu.be/xxxx
1006                             vid\.plus|                                        # or vid.plus/xxxx
1007                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
1008                             %(invidious)s
1009                          )/
1010                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1011                          )
1012                      )?                                                       # all until now is optional -> you can pass the naked ID
1013                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1014                      (?(1).+)?                                                # if we found the ID, everything can follow
1015                      (?:\#|$)""" % {
1016         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1017     }
1018     _EMBED_REGEX = [
1019         r'''(?x)
1020             (?:
1021                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1022                 data-video-url=|
1023                 <embed[^>]+?src=|
1024                 embedSWF\(?:\s*|
1025                 <object[^>]+data=|
1026                 new\s+SWFObject\(
1027             )
1028             (["\'])
1029                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1030                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1031             \1''',
1032         # https://wordpress.org/plugins/lazy-load-for-videos/
1033         r'''(?xs)
1034             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1035             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1036     ]
1037
1038     _PLAYER_INFO_RE = (
1039         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1040         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1041         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1042     )
1043     _formats = {
1044         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1045         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1046         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1047         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1048         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1049         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1050         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1051         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1052         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1053         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1054         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1055         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1056         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1057         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1058         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1059         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1060         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1061         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1062
1063
1064         # 3D videos
1065         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1066         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1067         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1068         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1069         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1070         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1071         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1072
1073         # Apple HTTP Live Streaming
1074         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1075         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1076         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1077         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1078         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1079         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1080         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1081         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1082
1083         # DASH mp4 video
1084         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1085         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1086         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1087         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1088         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1089         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1090         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1091         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1092         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1093         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1094         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1095         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1096
1097         # Dash mp4 audio
1098         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1099         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1100         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1101         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1102         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1103         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1104         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1105
1106         # Dash webm
1107         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1108         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1109         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1110         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1111         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1112         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1113         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1114         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1115         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1116         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1117         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1118         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1119         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1120         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1121         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1122         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1123         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1124         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1125         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1126         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1127         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1128         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1129
1130         # Dash webm audio
1131         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1132         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1133
1134         # Dash webm audio with opus inside
1135         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1136         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1137         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1138
1139         # RTMP (unnamed)
1140         '_rtmp': {'protocol': 'rtmp'},
1141
1142         # av01 video only formats sometimes served with "unknown" codecs
1143         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1144         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1145         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1146         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1147         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1148         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1149         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1150         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1151     }
1152     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1153
1154     _GEO_BYPASS = False
1155
1156     IE_NAME = 'youtube'
1157     _TESTS = [
1158         {
1159             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1160             'info_dict': {
1161                 'id': 'BaW_jenozKc',
1162                 'ext': 'mp4',
1163                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1164                 'uploader': 'Philipp Hagemeister',
1165                 'uploader_id': 'phihag',
1166                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1167                 'channel': 'Philipp Hagemeister',
1168                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1169                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1170                 'upload_date': '20121002',
1171                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1172                 'categories': ['Science & Technology'],
1173                 'tags': ['youtube-dl'],
1174                 'duration': 10,
1175                 'view_count': int,
1176                 'like_count': int,
1177                 'availability': 'public',
1178                 'playable_in_embed': True,
1179                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1180                 'live_status': 'not_live',
1181                 'age_limit': 0,
1182                 'start_time': 1,
1183                 'end_time': 9,
1184                 'comment_count': int,
1185                 'channel_follower_count': int
1186             }
1187         },
1188         {
1189             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1190             'note': 'Embed-only video (#1746)',
1191             'info_dict': {
1192                 'id': 'yZIXLfi8CZQ',
1193                 'ext': 'mp4',
1194                 'upload_date': '20120608',
1195                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1196                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1197                 'uploader': 'SET India',
1198                 'uploader_id': 'setindia',
1199                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1200                 'age_limit': 18,
1201             },
1202             'skip': 'Private video',
1203         },
1204         {
1205             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1206             'note': 'Use the first video ID in the URL',
1207             'info_dict': {
1208                 'id': 'BaW_jenozKc',
1209                 'ext': 'mp4',
1210                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1211                 'uploader': 'Philipp Hagemeister',
1212                 'uploader_id': 'phihag',
1213                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1214                 'channel': 'Philipp Hagemeister',
1215                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1216                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1217                 'upload_date': '20121002',
1218                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1219                 'categories': ['Science & Technology'],
1220                 'tags': ['youtube-dl'],
1221                 'duration': 10,
1222                 'view_count': int,
1223                 'like_count': int,
1224                 'availability': 'public',
1225                 'playable_in_embed': True,
1226                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1227                 'live_status': 'not_live',
1228                 'age_limit': 0,
1229                 'comment_count': int,
1230                 'channel_follower_count': int
1231             },
1232             'params': {
1233                 'skip_download': True,
1234             },
1235         },
1236         {
1237             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1238             'note': '256k DASH audio (format 141) via DASH manifest',
1239             'info_dict': {
1240                 'id': 'a9LDPn-MO4I',
1241                 'ext': 'm4a',
1242                 'upload_date': '20121002',
1243                 'uploader_id': '8KVIDEO',
1244                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1245                 'description': '',
1246                 'uploader': '8KVIDEO',
1247                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1248             },
1249             'params': {
1250                 'youtube_include_dash_manifest': True,
1251                 'format': '141',
1252             },
1253             'skip': 'format 141 not served anymore',
1254         },
1255         # DASH manifest with encrypted signature
1256         {
1257             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1258             'info_dict': {
1259                 'id': 'IB3lcPjvWLA',
1260                 'ext': 'm4a',
1261                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1262                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1263                 'duration': 244,
1264                 'uploader': 'AfrojackVEVO',
1265                 'uploader_id': 'AfrojackVEVO',
1266                 'upload_date': '20131011',
1267                 'abr': 129.495,
1268                 'like_count': int,
1269                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1270                 'playable_in_embed': True,
1271                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1272                 'view_count': int,
1273                 'track': 'The Spark',
1274                 'live_status': 'not_live',
1275                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1276                 'channel': 'Afrojack',
1277                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1278                 'tags': 'count:19',
1279                 'availability': 'public',
1280                 'categories': ['Music'],
1281                 'age_limit': 0,
1282                 'alt_title': 'The Spark',
1283                 'channel_follower_count': int
1284             },
1285             'params': {
1286                 'youtube_include_dash_manifest': True,
1287                 'format': '141/bestaudio[ext=m4a]',
1288             },
1289         },
1290         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1291         {
1292             'note': 'Embed allowed age-gate video',
1293             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1294             'info_dict': {
1295                 'id': 'HtVdAasjOgU',
1296                 'ext': 'mp4',
1297                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1298                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1299                 'duration': 142,
1300                 'uploader': 'The Witcher',
1301                 'uploader_id': 'WitcherGame',
1302                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1303                 'upload_date': '20140605',
1304                 'age_limit': 18,
1305                 'categories': ['Gaming'],
1306                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1307                 'availability': 'needs_auth',
1308                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1309                 'like_count': int,
1310                 'channel': 'The Witcher',
1311                 'live_status': 'not_live',
1312                 'tags': 'count:17',
1313                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1314                 'playable_in_embed': True,
1315                 'view_count': int,
1316                 'channel_follower_count': int
1317             },
1318         },
1319         {
1320             'note': 'Age-gate video with embed allowed in public site',
1321             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1322             'info_dict': {
1323                 'id': 'HsUATh_Nc2U',
1324                 'ext': 'mp4',
1325                 'title': 'Godzilla 2 (Official Video)',
1326                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1327                 'upload_date': '20200408',
1328                 'uploader_id': 'FlyingKitty900',
1329                 'uploader': 'FlyingKitty',
1330                 'age_limit': 18,
1331                 'availability': 'needs_auth',
1332                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1333                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1334                 'channel': 'FlyingKitty',
1335                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1336                 'view_count': int,
1337                 'categories': ['Entertainment'],
1338                 'live_status': 'not_live',
1339                 'tags': ['Flyingkitty', 'godzilla 2'],
1340                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1341                 'like_count': int,
1342                 'duration': 177,
1343                 'playable_in_embed': True,
1344                 'channel_follower_count': int
1345             },
1346         },
1347         {
1348             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1349             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1350             'info_dict': {
1351                 'id': 'Tq92D6wQ1mg',
1352                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1353                 'ext': 'mp4',
1354                 'upload_date': '20191228',
1355                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1356                 'uploader': 'Projekt Melody',
1357                 'description': 'md5:17eccca93a786d51bc67646756894066',
1358                 'age_limit': 18,
1359                 'like_count': int,
1360                 'availability': 'needs_auth',
1361                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1362                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1363                 'view_count': int,
1364                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1365                 'channel': 'Projekt Melody',
1366                 'live_status': 'not_live',
1367                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1368                 'playable_in_embed': True,
1369                 'categories': ['Entertainment'],
1370                 'duration': 106,
1371                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1372                 'comment_count': int,
1373                 'channel_follower_count': int
1374             },
1375         },
1376         {
1377             'note': 'Non-Agegated non-embeddable video',
1378             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1379             'info_dict': {
1380                 'id': 'MeJVWBSsPAY',
1381                 'ext': 'mp4',
1382                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1383                 'uploader': 'Herr Lurik',
1384                 'uploader_id': 'st3in234',
1385                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1386                 'upload_date': '20130730',
1387                 'track': 'Such mich find mich',
1388                 'age_limit': 0,
1389                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1390                 'like_count': int,
1391                 'playable_in_embed': False,
1392                 'creator': 'OOMPH!',
1393                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1394                 'view_count': int,
1395                 'alt_title': 'Such mich find mich',
1396                 'duration': 210,
1397                 'channel': 'Herr Lurik',
1398                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1399                 'categories': ['Music'],
1400                 'availability': 'public',
1401                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1402                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1403                 'live_status': 'not_live',
1404                 'artist': 'OOMPH!',
1405                 'channel_follower_count': int
1406             },
1407         },
1408         {
1409             'note': 'Non-bypassable age-gated video',
1410             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1411             'only_matching': True,
1412         },
1413         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1414         # YouTube Red ad is not captured for creator
1415         {
1416             'url': '__2ABJjxzNo',
1417             'info_dict': {
1418                 'id': '__2ABJjxzNo',
1419                 'ext': 'mp4',
1420                 'duration': 266,
1421                 'upload_date': '20100430',
1422                 'uploader_id': 'deadmau5',
1423                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1424                 'creator': 'deadmau5',
1425                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1426                 'uploader': 'deadmau5',
1427                 'title': 'Deadmau5 - Some Chords (HD)',
1428                 'alt_title': 'Some Chords',
1429                 'availability': 'public',
1430                 'tags': 'count:14',
1431                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1432                 'view_count': int,
1433                 'live_status': 'not_live',
1434                 'channel': 'deadmau5',
1435                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1436                 'like_count': int,
1437                 'track': 'Some Chords',
1438                 'artist': 'deadmau5',
1439                 'playable_in_embed': True,
1440                 'age_limit': 0,
1441                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1442                 'categories': ['Music'],
1443                 'album': 'Some Chords',
1444                 'channel_follower_count': int
1445             },
1446             'expected_warnings': [
1447                 'DASH manifest missing',
1448             ]
1449         },
1450         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1451         {
1452             'url': 'lqQg6PlCWgI',
1453             'info_dict': {
1454                 'id': 'lqQg6PlCWgI',
1455                 'ext': 'mp4',
1456                 'duration': 6085,
1457                 'upload_date': '20150827',
1458                 'uploader_id': 'olympic',
1459                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1460                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1461                 'uploader': 'Olympics',
1462                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1463                 'like_count': int,
1464                 'release_timestamp': 1343767800,
1465                 'playable_in_embed': True,
1466                 'categories': ['Sports'],
1467                 'release_date': '20120731',
1468                 'channel': 'Olympics',
1469                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1470                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1471                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1472                 'age_limit': 0,
1473                 'availability': 'public',
1474                 'live_status': 'was_live',
1475                 'view_count': int,
1476                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1477                 'channel_follower_count': int
1478             },
1479             'params': {
1480                 'skip_download': 'requires avconv',
1481             }
1482         },
1483         # Non-square pixels
1484         {
1485             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1486             'info_dict': {
1487                 'id': '_b-2C3KPAM0',
1488                 'ext': 'mp4',
1489                 'stretched_ratio': 16 / 9.,
1490                 'duration': 85,
1491                 'upload_date': '20110310',
1492                 'uploader_id': 'AllenMeow',
1493                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1494                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1495                 'uploader': '孫ᄋᄅ',
1496                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1497                 'playable_in_embed': True,
1498                 'channel': '孫ᄋᄅ',
1499                 'age_limit': 0,
1500                 'tags': 'count:11',
1501                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1502                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1503                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1504                 'view_count': int,
1505                 'categories': ['People & Blogs'],
1506                 'like_count': int,
1507                 'live_status': 'not_live',
1508                 'availability': 'unlisted',
1509                 'comment_count': int,
1510                 'channel_follower_count': int
1511             },
1512         },
1513         # url_encoded_fmt_stream_map is empty string
1514         {
1515             'url': 'qEJwOuvDf7I',
1516             'info_dict': {
1517                 'id': 'qEJwOuvDf7I',
1518                 'ext': 'webm',
1519                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1520                 'description': '',
1521                 'upload_date': '20150404',
1522                 'uploader_id': 'spbelect',
1523                 'uploader': 'Наблюдатели Петербурга',
1524             },
1525             'params': {
1526                 'skip_download': 'requires avconv',
1527             },
1528             'skip': 'This live event has ended.',
1529         },
1530         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1531         {
1532             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1533             'info_dict': {
1534                 'id': 'FIl7x6_3R5Y',
1535                 'ext': 'webm',
1536                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1537                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1538                 'duration': 220,
1539                 'upload_date': '20150625',
1540                 'uploader_id': 'dorappi2000',
1541                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1542                 'uploader': 'dorappi2000',
1543                 'formats': 'mincount:31',
1544             },
1545             'skip': 'not actual anymore',
1546         },
1547         # DASH manifest with segment_list
1548         {
1549             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1550             'md5': '8ce563a1d667b599d21064e982ab9e31',
1551             'info_dict': {
1552                 'id': 'CsmdDsKjzN8',
1553                 'ext': 'mp4',
1554                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1555                 'uploader': 'Airtek',
1556                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1557                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1558                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1559             },
1560             'params': {
1561                 'youtube_include_dash_manifest': True,
1562                 'format': '135',  # bestvideo
1563             },
1564             'skip': 'This live event has ended.',
1565         },
1566         {
1567             # Multifeed videos (multiple cameras), URL is for Main Camera
1568             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1569             'info_dict': {
1570                 'id': 'jvGDaLqkpTg',
1571                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1572                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1573             },
1574             'playlist': [{
1575                 'info_dict': {
1576                     'id': 'jvGDaLqkpTg',
1577                     'ext': 'mp4',
1578                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1579                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1580                     'duration': 10643,
1581                     'upload_date': '20161111',
1582                     'uploader': 'Team PGP',
1583                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1584                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1585                 },
1586             }, {
1587                 'info_dict': {
1588                     'id': '3AKt1R1aDnw',
1589                     'ext': 'mp4',
1590                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1591                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1592                     'duration': 10991,
1593                     'upload_date': '20161111',
1594                     'uploader': 'Team PGP',
1595                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1596                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1597                 },
1598             }, {
1599                 'info_dict': {
1600                     'id': 'RtAMM00gpVc',
1601                     'ext': 'mp4',
1602                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1603                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1604                     'duration': 10995,
1605                     'upload_date': '20161111',
1606                     'uploader': 'Team PGP',
1607                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1608                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1609                 },
1610             }, {
1611                 'info_dict': {
1612                     'id': '6N2fdlP3C5U',
1613                     'ext': 'mp4',
1614                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1615                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1616                     'duration': 10990,
1617                     'upload_date': '20161111',
1618                     'uploader': 'Team PGP',
1619                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1620                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1621                 },
1622             }],
1623             'params': {
1624                 'skip_download': True,
1625             },
1626             'skip': 'Not multifeed anymore',
1627         },
1628         {
1629             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1630             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1631             'info_dict': {
1632                 'id': 'gVfLd0zydlo',
1633                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1634             },
1635             'playlist_count': 2,
1636             'skip': 'Not multifeed anymore',
1637         },
1638         {
1639             'url': 'https://vid.plus/FlRa-iH7PGw',
1640             'only_matching': True,
1641         },
1642         {
1643             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1644             'only_matching': True,
1645         },
1646         {
1647             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1648             # Also tests cut-off URL expansion in video description (see
1649             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1650             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1651             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1652             'info_dict': {
1653                 'id': 'lsguqyKfVQg',
1654                 'ext': 'mp4',
1655                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1656                 'alt_title': 'Dark Walk',
1657                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1658                 'duration': 133,
1659                 'upload_date': '20151119',
1660                 'uploader_id': 'IronSoulElf',
1661                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1662                 'uploader': 'IronSoulElf',
1663                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1664                 'track': 'Dark Walk',
1665                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1666                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1667                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1668                 'categories': ['Film & Animation'],
1669                 'view_count': int,
1670                 'live_status': 'not_live',
1671                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1672                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1673                 'tags': 'count:13',
1674                 'availability': 'public',
1675                 'channel': 'IronSoulElf',
1676                 'playable_in_embed': True,
1677                 'like_count': int,
1678                 'age_limit': 0,
1679                 'channel_follower_count': int
1680             },
1681             'params': {
1682                 'skip_download': True,
1683             },
1684         },
1685         {
1686             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1687             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1688             'only_matching': True,
1689         },
1690         {
1691             # Video with yt:stretch=17:0
1692             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1693             'info_dict': {
1694                 'id': 'Q39EVAstoRM',
1695                 'ext': 'mp4',
1696                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1697                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1698                 'upload_date': '20151107',
1699                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1700                 'uploader': 'CH GAMER DROID',
1701             },
1702             'params': {
1703                 'skip_download': True,
1704             },
1705             'skip': 'This video does not exist.',
1706         },
1707         {
1708             # Video with incomplete 'yt:stretch=16:'
1709             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1710             'only_matching': True,
1711         },
1712         {
1713             # Video licensed under Creative Commons
1714             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1715             'info_dict': {
1716                 'id': 'M4gD1WSo5mA',
1717                 'ext': 'mp4',
1718                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1719                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1720                 'duration': 721,
1721                 'upload_date': '20150128',
1722                 'uploader_id': 'BerkmanCenter',
1723                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1724                 'uploader': 'The Berkman Klein Center for Internet & Society',
1725                 'license': 'Creative Commons Attribution license (reuse allowed)',
1726                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1727                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1728                 'like_count': int,
1729                 'age_limit': 0,
1730                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1731                 'channel': 'The Berkman Klein Center for Internet & Society',
1732                 'availability': 'public',
1733                 'view_count': int,
1734                 'categories': ['Education'],
1735                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1736                 'live_status': 'not_live',
1737                 'playable_in_embed': True,
1738                 'comment_count': int,
1739                 'channel_follower_count': int,
1740                 'chapters': list,
1741             },
1742             'params': {
1743                 'skip_download': True,
1744             },
1745         },
1746         {
1747             # Channel-like uploader_url
1748             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1749             'info_dict': {
1750                 'id': 'eQcmzGIKrzg',
1751                 'ext': 'mp4',
1752                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1753                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1754                 'duration': 4060,
1755                 'upload_date': '20151120',
1756                 'uploader': 'Bernie Sanders',
1757                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1758                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1759                 'license': 'Creative Commons Attribution license (reuse allowed)',
1760                 'playable_in_embed': True,
1761                 'tags': 'count:12',
1762                 'like_count': int,
1763                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1764                 'age_limit': 0,
1765                 'availability': 'public',
1766                 'categories': ['News & Politics'],
1767                 'channel': 'Bernie Sanders',
1768                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1769                 'view_count': int,
1770                 'live_status': 'not_live',
1771                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1772                 'comment_count': int,
1773                 'channel_follower_count': int,
1774                 'chapters': list,
1775             },
1776             'params': {
1777                 'skip_download': True,
1778             },
1779         },
1780         {
1781             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1782             'only_matching': True,
1783         },
1784         {
1785             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1786             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1787             'only_matching': True,
1788         },
1789         {
1790             # Rental video preview
1791             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1792             'info_dict': {
1793                 'id': 'uGpuVWrhIzE',
1794                 'ext': 'mp4',
1795                 'title': 'Piku - Trailer',
1796                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1797                 'upload_date': '20150811',
1798                 'uploader': 'FlixMatrix',
1799                 'uploader_id': 'FlixMatrixKaravan',
1800                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1801                 'license': 'Standard YouTube License',
1802             },
1803             'params': {
1804                 'skip_download': True,
1805             },
1806             'skip': 'This video is not available.',
1807         },
1808         {
1809             # YouTube Red video with episode data
1810             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1811             'info_dict': {
1812                 'id': 'iqKdEhx-dD4',
1813                 'ext': 'mp4',
1814                 'title': 'Isolation - Mind Field (Ep 1)',
1815                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1816                 'duration': 2085,
1817                 'upload_date': '20170118',
1818                 'uploader': 'Vsauce',
1819                 'uploader_id': 'Vsauce',
1820                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1821                 'series': 'Mind Field',
1822                 'season_number': 1,
1823                 'episode_number': 1,
1824                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1825                 'tags': 'count:12',
1826                 'view_count': int,
1827                 'availability': 'public',
1828                 'age_limit': 0,
1829                 'channel': 'Vsauce',
1830                 'episode': 'Episode 1',
1831                 'categories': ['Entertainment'],
1832                 'season': 'Season 1',
1833                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1834                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1835                 'like_count': int,
1836                 'playable_in_embed': True,
1837                 'live_status': 'not_live',
1838                 'channel_follower_count': int
1839             },
1840             'params': {
1841                 'skip_download': True,
1842             },
1843             'expected_warnings': [
1844                 'Skipping DASH manifest',
1845             ],
1846         },
1847         {
1848             # The following content has been identified by the YouTube community
1849             # as inappropriate or offensive to some audiences.
1850             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1851             'info_dict': {
1852                 'id': '6SJNVb0GnPI',
1853                 'ext': 'mp4',
1854                 'title': 'Race Differences in Intelligence',
1855                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1856                 'duration': 965,
1857                 'upload_date': '20140124',
1858                 'uploader': 'New Century Foundation',
1859                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1860                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1861             },
1862             'params': {
1863                 'skip_download': True,
1864             },
1865             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1866         },
1867         {
1868             # itag 212
1869             'url': '1t24XAntNCY',
1870             'only_matching': True,
1871         },
1872         {
1873             # geo restricted to JP
1874             'url': 'sJL6WA-aGkQ',
1875             'only_matching': True,
1876         },
1877         {
1878             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1879             'only_matching': True,
1880         },
1881         {
1882             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1883             'only_matching': True,
1884         },
1885         {
1886             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1887             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1888             'only_matching': True,
1889         },
1890         {
1891             # DRM protected
1892             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1893             'only_matching': True,
1894         },
1895         {
1896             # Video with unsupported adaptive stream type formats
1897             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1898             'info_dict': {
1899                 'id': 'Z4Vy8R84T1U',
1900                 'ext': 'mp4',
1901                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1902                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1903                 'duration': 433,
1904                 'upload_date': '20130923',
1905                 'uploader': 'Amelia Putri Harwita',
1906                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1907                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1908                 'formats': 'maxcount:10',
1909             },
1910             'params': {
1911                 'skip_download': True,
1912                 'youtube_include_dash_manifest': False,
1913             },
1914             'skip': 'not actual anymore',
1915         },
1916         {
1917             # Youtube Music Auto-generated description
1918             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1919             'info_dict': {
1920                 'id': 'MgNrAu2pzNs',
1921                 'ext': 'mp4',
1922                 'title': 'Voyeur Girl',
1923                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1924                 'upload_date': '20190312',
1925                 'uploader': 'Stephen - Topic',
1926                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1927                 'artist': 'Stephen',
1928                 'track': 'Voyeur Girl',
1929                 'album': 'it\'s too much love to know my dear',
1930                 'release_date': '20190313',
1931                 'release_year': 2019,
1932                 'alt_title': 'Voyeur Girl',
1933                 'view_count': int,
1934                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1935                 'playable_in_embed': True,
1936                 'like_count': int,
1937                 'categories': ['Music'],
1938                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1939                 'channel': 'Stephen',
1940                 'availability': 'public',
1941                 'creator': 'Stephen',
1942                 'duration': 169,
1943                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1944                 'age_limit': 0,
1945                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1946                 'tags': 'count:11',
1947                 'live_status': 'not_live',
1948                 'channel_follower_count': int
1949             },
1950             'params': {
1951                 'skip_download': True,
1952             },
1953         },
1954         {
1955             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1956             'only_matching': True,
1957         },
1958         {
1959             # invalid -> valid video id redirection
1960             'url': 'DJztXj2GPfl',
1961             'info_dict': {
1962                 'id': 'DJztXj2GPfk',
1963                 'ext': 'mp4',
1964                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1965                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1966                 'upload_date': '20090125',
1967                 'uploader': 'Prochorowka',
1968                 'uploader_id': 'Prochorowka',
1969                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1970                 'artist': 'Panjabi MC',
1971                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1972                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1973             },
1974             'params': {
1975                 'skip_download': True,
1976             },
1977             'skip': 'Video unavailable',
1978         },
1979         {
1980             # empty description results in an empty string
1981             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1982             'info_dict': {
1983                 'id': 'x41yOUIvK2k',
1984                 'ext': 'mp4',
1985                 'title': 'IMG 3456',
1986                 'description': '',
1987                 'upload_date': '20170613',
1988                 'uploader_id': 'ElevageOrVert',
1989                 'uploader': 'ElevageOrVert',
1990                 'view_count': int,
1991                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1992                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1993                 'like_count': int,
1994                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1995                 'tags': [],
1996                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1997                 'availability': 'public',
1998                 'age_limit': 0,
1999                 'categories': ['Pets & Animals'],
2000                 'duration': 7,
2001                 'playable_in_embed': True,
2002                 'live_status': 'not_live',
2003                 'channel': 'ElevageOrVert',
2004                 'channel_follower_count': int
2005             },
2006             'params': {
2007                 'skip_download': True,
2008             },
2009         },
2010         {
2011             # with '};' inside yt initial data (see [1])
2012             # see [2] for an example with '};' inside ytInitialPlayerResponse
2013             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2014             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2015             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2016             'info_dict': {
2017                 'id': 'CHqg6qOn4no',
2018                 'ext': 'mp4',
2019                 'title': 'Part 77   Sort a list of simple types in c#',
2020                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2021                 'upload_date': '20130831',
2022                 'uploader_id': 'kudvenkat',
2023                 'uploader': 'kudvenkat',
2024                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2025                 'like_count': int,
2026                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2027                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2028                 'live_status': 'not_live',
2029                 'categories': ['Education'],
2030                 'availability': 'public',
2031                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2032                 'tags': 'count:12',
2033                 'playable_in_embed': True,
2034                 'age_limit': 0,
2035                 'view_count': int,
2036                 'duration': 522,
2037                 'channel': 'kudvenkat',
2038                 'comment_count': int,
2039                 'channel_follower_count': int,
2040                 'chapters': list,
2041             },
2042             'params': {
2043                 'skip_download': True,
2044             },
2045         },
2046         {
2047             # another example of '};' in ytInitialData
2048             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2049             'only_matching': True,
2050         },
2051         {
2052             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2053             'only_matching': True,
2054         },
2055         {
2056             # https://github.com/ytdl-org/youtube-dl/pull/28094
2057             'url': 'OtqTfy26tG0',
2058             'info_dict': {
2059                 'id': 'OtqTfy26tG0',
2060                 'ext': 'mp4',
2061                 'title': 'Burn Out',
2062                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2063                 'upload_date': '20141120',
2064                 'uploader': 'The Cinematic Orchestra - Topic',
2065                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2066                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2067                 'artist': 'The Cinematic Orchestra',
2068                 'track': 'Burn Out',
2069                 'album': 'Every Day',
2070                 'like_count': int,
2071                 'live_status': 'not_live',
2072                 'alt_title': 'Burn Out',
2073                 'duration': 614,
2074                 'age_limit': 0,
2075                 'view_count': int,
2076                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2077                 'creator': 'The Cinematic Orchestra',
2078                 'channel': 'The Cinematic Orchestra',
2079                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2080                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2081                 'availability': 'public',
2082                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2083                 'categories': ['Music'],
2084                 'playable_in_embed': True,
2085                 'channel_follower_count': int
2086             },
2087             'params': {
2088                 'skip_download': True,
2089             },
2090         },
2091         {
2092             # controversial video, only works with bpctr when authenticated with cookies
2093             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2094             'only_matching': True,
2095         },
2096         {
2097             # controversial video, requires bpctr/contentCheckOk
2098             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2099             'info_dict': {
2100                 'id': 'SZJvDhaSDnc',
2101                 'ext': 'mp4',
2102                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2103                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2104                 'uploader': 'CBS Mornings',
2105                 'uploader_id': 'CBSThisMorning',
2106                 'upload_date': '20140716',
2107                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2108                 'duration': 170,
2109                 'categories': ['News & Politics'],
2110                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2111                 'view_count': int,
2112                 'channel': 'CBS Mornings',
2113                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2114                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2115                 'age_limit': 18,
2116                 'availability': 'needs_auth',
2117                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2118                 'like_count': int,
2119                 'live_status': 'not_live',
2120                 'playable_in_embed': True,
2121                 'channel_follower_count': int
2122             }
2123         },
2124         {
2125             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2126             'url': 'cBvYw8_A0vQ',
2127             'info_dict': {
2128                 'id': 'cBvYw8_A0vQ',
2129                 'ext': 'mp4',
2130                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2131                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2132                 'upload_date': '20201120',
2133                 'uploader': 'Walk around Japan',
2134                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2135                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2136                 'duration': 1456,
2137                 'categories': ['Travel & Events'],
2138                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2139                 'view_count': int,
2140                 'channel': 'Walk around Japan',
2141                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2142                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2143                 'age_limit': 0,
2144                 'availability': 'public',
2145                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2146                 'live_status': 'not_live',
2147                 'playable_in_embed': True,
2148                 'channel_follower_count': int
2149             },
2150             'params': {
2151                 'skip_download': True,
2152             },
2153         }, {
2154             # Has multiple audio streams
2155             'url': 'WaOKSUlf4TM',
2156             'only_matching': True
2157         }, {
2158             # Requires Premium: has format 141 when requested using YTM url
2159             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2160             'only_matching': True
2161         }, {
2162             # multiple subtitles with same lang_code
2163             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2164             'only_matching': True,
2165         }, {
2166             # Force use android client fallback
2167             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2168             'info_dict': {
2169                 'id': 'YOelRv7fMxY',
2170                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2171                 'ext': '3gp',
2172                 'upload_date': '20210624',
2173                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2174                 'uploader': 'colinfurze',
2175                 'uploader_id': 'colinfurze',
2176                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2177                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2178                 'duration': 596,
2179                 'categories': ['Entertainment'],
2180                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2181                 'view_count': int,
2182                 'channel': 'colinfurze',
2183                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2184                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2185                 'age_limit': 0,
2186                 'availability': 'public',
2187                 'like_count': int,
2188                 'live_status': 'not_live',
2189                 'playable_in_embed': True,
2190                 'channel_follower_count': int,
2191                 'chapters': list,
2192             },
2193             'params': {
2194                 'format': '17',  # 3gp format available on android
2195                 'extractor_args': {'youtube': {'player_client': ['android']}},
2196             },
2197         },
2198         {
2199             # Skip download of additional client configs (remix client config in this case)
2200             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2201             'only_matching': True,
2202             'params': {
2203                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2204             },
2205         }, {
2206             # shorts
2207             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2208             'only_matching': True,
2209         }, {
2210             'note': 'Storyboards',
2211             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2212             'info_dict': {
2213                 'id': '5KLPxDtMqe8',
2214                 'ext': 'mhtml',
2215                 'format_id': 'sb0',
2216                 'title': 'Your Brain is Plastic',
2217                 'uploader_id': 'scishow',
2218                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2219                 'upload_date': '20140324',
2220                 'uploader': 'SciShow',
2221                 'like_count': int,
2222                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2223                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2224                 'view_count': int,
2225                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2226                 'playable_in_embed': True,
2227                 'tags': 'count:12',
2228                 'uploader_url': 'http://www.youtube.com/user/scishow',
2229                 'availability': 'public',
2230                 'channel': 'SciShow',
2231                 'live_status': 'not_live',
2232                 'duration': 248,
2233                 'categories': ['Education'],
2234                 'age_limit': 0,
2235                 'channel_follower_count': int,
2236                 'chapters': list,
2237             }, 'params': {'format': 'mhtml', 'skip_download': True}
2238         }, {
2239             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2240             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2241             'info_dict': {
2242                 'id': '2NUZ8W2llS4',
2243                 'ext': 'mp4',
2244                 'title': 'The NP that test your phone performance 🙂',
2245                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2246                 'uploader': 'Leon Nguyen',
2247                 'uploader_id': 'VNSXIII',
2248                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2249                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2250                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2251                 'duration': 21,
2252                 'view_count': int,
2253                 'age_limit': 0,
2254                 'categories': ['Gaming'],
2255                 'tags': 'count:23',
2256                 'playable_in_embed': True,
2257                 'live_status': 'not_live',
2258                 'upload_date': '20220103',
2259                 'like_count': int,
2260                 'availability': 'public',
2261                 'channel': 'Leon Nguyen',
2262                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2263                 'comment_count': int,
2264                 'channel_follower_count': int
2265             }
2266         }, {
2267             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2268             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2269             'info_dict': {
2270                 'id': '2NUZ8W2llS4',
2271                 'ext': 'mp4',
2272                 'title': 'The NP that test your phone performance 🙂',
2273                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2274                 'uploader': 'Leon Nguyen',
2275                 'uploader_id': 'VNSXIII',
2276                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2277                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2278                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2279                 'duration': 21,
2280                 'view_count': int,
2281                 'age_limit': 0,
2282                 'categories': ['Gaming'],
2283                 'tags': 'count:23',
2284                 'playable_in_embed': True,
2285                 'live_status': 'not_live',
2286                 'upload_date': '20220102',
2287                 'like_count': int,
2288                 'availability': 'public',
2289                 'channel': 'Leon Nguyen',
2290                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2291                 'comment_count': int,
2292                 'channel_follower_count': int
2293             },
2294             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2295         }, {
2296             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2297             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2298             'info_dict': {
2299                 'id': 'mzZzzBU6lrM',
2300                 'ext': 'mp4',
2301                 'title': 'I Met GeorgeNotFound In Real Life...',
2302                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2303                 'uploader': 'Quackity',
2304                 'uploader_id': 'QuackityHQ',
2305                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2306                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2307                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2308                 'duration': 955,
2309                 'view_count': int,
2310                 'age_limit': 0,
2311                 'categories': ['Entertainment'],
2312                 'tags': 'count:26',
2313                 'playable_in_embed': True,
2314                 'live_status': 'not_live',
2315                 'release_timestamp': 1641172509,
2316                 'release_date': '20220103',
2317                 'upload_date': '20220103',
2318                 'like_count': int,
2319                 'availability': 'public',
2320                 'channel': 'Quackity',
2321                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2322                 'channel_follower_count': int
2323             }
2324         },
2325         {   # continuous livestream. Microformat upload date should be preferred.
2326             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2327             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2328             'info_dict': {
2329                 'id': 'kgx4WGK0oNU',
2330                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2331                 'ext': 'mp4',
2332                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2333                 'availability': 'public',
2334                 'age_limit': 0,
2335                 'release_timestamp': 1637975704,
2336                 'upload_date': '20210619',
2337                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2338                 'live_status': 'is_live',
2339                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2340                 'uploader': '阿鲍Abao',
2341                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2342                 'channel': 'Abao in Tokyo',
2343                 'channel_follower_count': int,
2344                 'release_date': '20211127',
2345                 'tags': 'count:39',
2346                 'categories': ['People & Blogs'],
2347                 'like_count': int,
2348                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2349                 'view_count': int,
2350                 'playable_in_embed': True,
2351                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2352                 'concurrent_view_count': int,
2353             },
2354             'params': {'skip_download': True}
2355         }, {
2356             # Story. Requires specific player params to work.
2357             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2358             'info_dict': {
2359                 'id': 'vv8qTUWmulI',
2360                 'ext': 'mp4',
2361                 'availability': 'unlisted',
2362                 'view_count': int,
2363                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2364                 'upload_date': '20220526',
2365                 'categories': ['Education'],
2366                 'title': 'Story',
2367                 'channel': 'IT\'S HISTORY',
2368                 'description': '',
2369                 'uploader_id': 'BlastfromthePast',
2370                 'duration': 12,
2371                 'uploader': 'IT\'S HISTORY',
2372                 'playable_in_embed': True,
2373                 'age_limit': 0,
2374                 'live_status': 'not_live',
2375                 'tags': [],
2376                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2377                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2378                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2379             },
2380             'skip': 'stories get removed after some period of time',
2381         }, {
2382             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2383             'info_dict': {
2384                 'id': 'tjjjtzRLHvA',
2385                 'ext': 'mp4',
2386                 'title': 'ハッシュタグ無し };if window.ytcsi',
2387                 'upload_date': '20220323',
2388                 'like_count': int,
2389                 'availability': 'unlisted',
2390                 'channel': 'nao20010128nao',
2391                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2392                 'age_limit': 0,
2393                 'uploader': 'nao20010128nao',
2394                 'uploader_id': 'nao20010128nao',
2395                 'categories': ['Music'],
2396                 'view_count': int,
2397                 'description': '',
2398                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2399                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2400                 'live_status': 'not_live',
2401                 'playable_in_embed': True,
2402                 'channel_follower_count': int,
2403                 'duration': 6,
2404                 'tags': [],
2405                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2406             }
2407         }, {
2408             # Prefer primary title+description language metadata by default
2409             # Do not prefer translated description if primary is empty
2410             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2411             'info_dict': {
2412                 'id': 'el3E4MbxRqQ',
2413                 'ext': 'mp4',
2414                 'title': 'dlp test video 2 - primary sv no desc',
2415                 'description': '',
2416                 'channel': 'cole-dlp-test-acc',
2417                 'tags': [],
2418                 'view_count': int,
2419                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2420                 'like_count': int,
2421                 'playable_in_embed': True,
2422                 'availability': 'unlisted',
2423                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2424                 'age_limit': 0,
2425                 'duration': 5,
2426                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2427                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2428                 'live_status': 'not_live',
2429                 'upload_date': '20220908',
2430                 'categories': ['People & Blogs'],
2431                 'uploader': 'cole-dlp-test-acc',
2432                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2433             },
2434             'params': {'skip_download': True}
2435         }, {
2436             # Extractor argument: prefer translated title+description
2437             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2438             'info_dict': {
2439                 'id': 'gHKT4uU8Zng',
2440                 'ext': 'mp4',
2441                 'channel': 'cole-dlp-test-acc',
2442                 'tags': [],
2443                 'duration': 5,
2444                 'live_status': 'not_live',
2445                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2446                 'upload_date': '20220728',
2447                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2448                 'view_count': int,
2449                 'categories': ['People & Blogs'],
2450                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2451                 'title': 'dlp test video title translated (fr)',
2452                 'availability': 'public',
2453                 'uploader': 'cole-dlp-test-acc',
2454                 'age_limit': 0,
2455                 'description': 'dlp test video description translated (fr)',
2456                 'playable_in_embed': True,
2457                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2458                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2459             },
2460             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2461             'expected_warnings': [r'Preferring "fr" translated fields'],
2462         }, {
2463             'note': '6 channel audio',
2464             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2465             'only_matching': True,
2466         }
2467     ]
2468
2469     _WEBPAGE_TESTS = [
2470         # YouTube <object> embed
2471         {
2472             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2473             'md5': '873c81d308b979f0e23ee7e620b312a3',
2474             'info_dict': {
2475                 'id': 'msN87y-iEx0',
2476                 'ext': 'mp4',
2477                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2478                 'upload_date': '20080526',
2479                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2480                 'uploader': 'Christopher Sykes',
2481                 'uploader_id': 'ChristopherJSykes',
2482                 'age_limit': 0,
2483                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2484                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2485                 'playable_in_embed': True,
2486                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2487                 'like_count': int,
2488                 'comment_count': int,
2489                 'channel': 'Christopher Sykes',
2490                 'live_status': 'not_live',
2491                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2492                 'availability': 'public',
2493                 'duration': 195,
2494                 'view_count': int,
2495                 'categories': ['Science & Technology'],
2496                 'channel_follower_count': int,
2497                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2498             },
2499             'params': {
2500                 'skip_download': True,
2501             }
2502         },
2503     ]
2504
2505     @classmethod
2506     def suitable(cls, url):
2507         from ..utils import parse_qs
2508
2509         qs = parse_qs(url)
2510         if qs.get('list', [None])[0]:
2511             return False
2512         return super().suitable(url)
2513
2514     def __init__(self, *args, **kwargs):
2515         super().__init__(*args, **kwargs)
2516         self._code_cache = {}
2517         self._player_cache = {}
2518
2519     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2520         lock = threading.Lock()
2521         start_time = time.time()
2522         formats = [f for f in formats if f.get('is_from_start')]
2523
2524         def refetch_manifest(format_id, delay):
2525             nonlocal formats, start_time, is_live
2526             if time.time() <= start_time + delay:
2527                 return
2528
2529             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2530             video_details = traverse_obj(
2531                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2532             microformats = traverse_obj(
2533                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2534                 expected_type=dict, default=[])
2535             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2536             is_live = live_status == 'is_live'
2537             start_time = time.time()
2538
2539         def mpd_feed(format_id, delay):
2540             """
2541             @returns (manifest_url, manifest_stream_number, is_live) or None
2542             """
2543             with lock:
2544                 refetch_manifest(format_id, delay)
2545
2546             f = next((f for f in formats if f['format_id'] == format_id), None)
2547             if not f:
2548                 if not is_live:
2549                     self.to_screen(f'{video_id}: Video is no longer live')
2550                 else:
2551                     self.report_warning(
2552                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2553                 return None
2554             return f['manifest_url'], f['manifest_stream_number'], is_live
2555
2556         for f in formats:
2557             f['is_live'] = is_live
2558             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2559                                     live_start_time, mpd_feed, not is_live and f.copy())
2560             if is_live:
2561                 f['fragments'] = gen
2562                 f['protocol'] = 'http_dash_segments_generator'
2563             else:
2564                 f['fragments'] = LazyList(gen({}))
2565                 del f['is_from_start']
2566
2567     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2568         FETCH_SPAN, MAX_DURATION = 5, 432000
2569
2570         mpd_url, stream_number, is_live = None, None, True
2571
2572         begin_index = 0
2573         download_start_time = ctx.get('start') or time.time()
2574
2575         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2576         if lack_early_segments:
2577             self.report_warning(bug_reports_message(
2578                 'Starting download from the last 120 hours of the live stream since '
2579                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2580             lack_early_segments = True
2581
2582         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2583         fragments, fragment_base_url = None, None
2584
2585         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2586             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2587             # Obtain from MPD's maximum seq value
2588             old_mpd_url = mpd_url
2589             last_error = ctx.pop('last_error', None)
2590             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2591             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2592                                                or (mpd_url, stream_number, False))
2593             if not refresh_sequence:
2594                 if expire_fast and not is_live:
2595                     return False, last_seq
2596                 elif old_mpd_url == mpd_url:
2597                     return True, last_seq
2598             if manifestless_orig_fmt:
2599                 fmt_info = manifestless_orig_fmt
2600             else:
2601                 try:
2602                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2603                         mpd_url, None, note=False, errnote=False, fatal=False)
2604                 except ExtractorError:
2605                     fmts = None
2606                 if not fmts:
2607                     no_fragment_score += 2
2608                     return False, last_seq
2609                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2610             fragments = fmt_info['fragments']
2611             fragment_base_url = fmt_info['fragment_base_url']
2612             assert fragment_base_url
2613
2614             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2615             return True, _last_seq
2616
2617         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2618         while is_live:
2619             fetch_time = time.time()
2620             if no_fragment_score > 30:
2621                 return
2622             if last_segment_url:
2623                 # Obtain from "X-Head-Seqnum" header value from each segment
2624                 try:
2625                     urlh = self._request_webpage(
2626                         last_segment_url, None, note=False, errnote=False, fatal=False)
2627                 except ExtractorError:
2628                     urlh = None
2629                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2630                 if last_seq is None:
2631                     no_fragment_score += 2
2632                     last_segment_url = None
2633                     continue
2634             else:
2635                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2636                 no_fragment_score += 2
2637                 if not should_continue:
2638                     continue
2639
2640             if known_idx > last_seq:
2641                 last_segment_url = None
2642                 continue
2643
2644             last_seq += 1
2645
2646             if begin_index < 0 and known_idx < 0:
2647                 # skip from the start when it's negative value
2648                 known_idx = last_seq + begin_index
2649             if lack_early_segments:
2650                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2651             try:
2652                 for idx in range(known_idx, last_seq):
2653                     # do not update sequence here or you'll get skipped some part of it
2654                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2655                     if not should_continue:
2656                         known_idx = idx - 1
2657                         raise ExtractorError('breaking out of outer loop')
2658                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2659                     yield {
2660                         'url': last_segment_url,
2661                         'fragment_count': last_seq,
2662                     }
2663                 if known_idx == last_seq:
2664                     no_fragment_score += 5
2665                 else:
2666                     no_fragment_score = 0
2667                 known_idx = last_seq
2668             except ExtractorError:
2669                 continue
2670
2671             if manifestless_orig_fmt:
2672                 # Stop at the first iteration if running for post-live manifestless;
2673                 # fragment count no longer increase since it starts
2674                 break
2675
2676             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2677
2678     def _extract_player_url(self, *ytcfgs, webpage=None):
2679         player_url = traverse_obj(
2680             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2681             get_all=False, expected_type=str)
2682         if not player_url:
2683             return
2684         return urljoin('https://www.youtube.com', player_url)
2685
2686     def _download_player_url(self, video_id, fatal=False):
2687         res = self._download_webpage(
2688             'https://www.youtube.com/iframe_api',
2689             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2690         if res:
2691             player_version = self._search_regex(
2692                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2693             if player_version:
2694                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2695
2696     def _signature_cache_id(self, example_sig):
2697         """ Return a string representation of a signature """
2698         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2699
2700     @classmethod
2701     def _extract_player_info(cls, player_url):
2702         for player_re in cls._PLAYER_INFO_RE:
2703             id_m = re.search(player_re, player_url)
2704             if id_m:
2705                 break
2706         else:
2707             raise ExtractorError('Cannot identify player %r' % player_url)
2708         return id_m.group('id')
2709
2710     def _load_player(self, video_id, player_url, fatal=True):
2711         player_id = self._extract_player_info(player_url)
2712         if player_id not in self._code_cache:
2713             code = self._download_webpage(
2714                 player_url, video_id, fatal=fatal,
2715                 note='Downloading player ' + player_id,
2716                 errnote='Download of %s failed' % player_url)
2717             if code:
2718                 self._code_cache[player_id] = code
2719         return self._code_cache.get(player_id)
2720
2721     def _extract_signature_function(self, video_id, player_url, example_sig):
2722         player_id = self._extract_player_info(player_url)
2723
2724         # Read from filesystem cache
2725         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2726         assert os.path.basename(func_id) == func_id
2727
2728         self.write_debug(f'Extracting signature function {func_id}')
2729         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2730
2731         if not cache_spec:
2732             code = self._load_player(video_id, player_url)
2733         if code:
2734             res = self._parse_sig_js(code)
2735             test_string = ''.join(map(chr, range(len(example_sig))))
2736             cache_spec = [ord(c) for c in res(test_string)]
2737             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2738
2739         return lambda s: ''.join(s[i] for i in cache_spec)
2740
2741     def _print_sig_code(self, func, example_sig):
2742         if not self.get_param('youtube_print_sig_code'):
2743             return
2744
2745         def gen_sig_code(idxs):
2746             def _genslice(start, end, step):
2747                 starts = '' if start == 0 else str(start)
2748                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2749                 steps = '' if step == 1 else (':%d' % step)
2750                 return f's[{starts}{ends}{steps}]'
2751
2752             step = None
2753             # Quelch pyflakes warnings - start will be set when step is set
2754             start = '(Never used)'
2755             for i, prev in zip(idxs[1:], idxs[:-1]):
2756                 if step is not None:
2757                     if i - prev == step:
2758                         continue
2759                     yield _genslice(start, prev, step)
2760                     step = None
2761                     continue
2762                 if i - prev in [-1, 1]:
2763                     step = i - prev
2764                     start = prev
2765                     continue
2766                 else:
2767                     yield 's[%d]' % prev
2768             if step is None:
2769                 yield 's[%d]' % i
2770             else:
2771                 yield _genslice(start, i, step)
2772
2773         test_string = ''.join(map(chr, range(len(example_sig))))
2774         cache_res = func(test_string)
2775         cache_spec = [ord(c) for c in cache_res]
2776         expr_code = ' + '.join(gen_sig_code(cache_spec))
2777         signature_id_tuple = '(%s)' % (
2778             ', '.join(str(len(p)) for p in example_sig.split('.')))
2779         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2780                 '    return %s\n') % (signature_id_tuple, expr_code)
2781         self.to_screen('Extracted signature function:\n' + code)
2782
2783     def _parse_sig_js(self, jscode):
2784         funcname = self._search_regex(
2785             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2786              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2787              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2788              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2789              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2790              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2791              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2792              # Obsolete patterns
2793              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2794              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2795              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2796              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2797              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2798              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2799              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2800              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2801             jscode, 'Initial JS player signature function name', group='sig')
2802
2803         jsi = JSInterpreter(jscode)
2804         initial_function = jsi.extract_function(funcname)
2805         return lambda s: initial_function([s])
2806
2807     def _cached(self, func, *cache_id):
2808         def inner(*args, **kwargs):
2809             if cache_id not in self._player_cache:
2810                 try:
2811                     self._player_cache[cache_id] = func(*args, **kwargs)
2812                 except ExtractorError as e:
2813                     self._player_cache[cache_id] = e
2814                 except Exception as e:
2815                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
2816
2817             ret = self._player_cache[cache_id]
2818             if isinstance(ret, Exception):
2819                 raise ret
2820             return ret
2821         return inner
2822
2823     def _decrypt_signature(self, s, video_id, player_url):
2824         """Turn the encrypted s field into a working signature"""
2825         extract_sig = self._cached(
2826             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
2827         func = extract_sig(video_id, player_url, s)
2828         self._print_sig_code(func, s)
2829         return func(s)
2830
2831     def _decrypt_nsig(self, s, video_id, player_url):
2832         """Turn the encrypted n field into a working signature"""
2833         if player_url is None:
2834             raise ExtractorError('Cannot decrypt nsig without player_url')
2835         player_url = urljoin('https://www.youtube.com', player_url)
2836
2837         try:
2838             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
2839         except ExtractorError as e:
2840             raise ExtractorError('Unable to extract nsig function code', cause=e)
2841         if self.get_param('youtube_print_sig_code'):
2842             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2843
2844         try:
2845             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
2846             ret = extract_nsig(jsi, func_code)(s)
2847         except JSInterpreter.Exception as e:
2848             try:
2849                 jsi = PhantomJSwrapper(self, timeout=5000)
2850             except ExtractorError:
2851                 raise e
2852             self.report_warning(
2853                 f'Native nsig extraction failed: Trying with PhantomJS\n'
2854                 f'         n = {s} ; player = {player_url}', video_id)
2855             self.write_debug(e, only_once=True)
2856
2857             args, func_body = func_code
2858             ret = jsi.execute(
2859                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
2860                 video_id=video_id, note='Executing signature code').strip()
2861
2862         self.write_debug(f'Decrypted nsig {s} => {ret}')
2863         return ret
2864
2865     def _extract_n_function_name(self, jscode):
2866         funcname, idx = self._search_regex(
2867             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2868             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2869         if not idx:
2870             return funcname
2871
2872         return json.loads(js_to_json(self._search_regex(
2873             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
2874             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
2875
2876     def _extract_n_function_code(self, video_id, player_url):
2877         player_id = self._extract_player_info(player_url)
2878         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
2879         jscode = func_code or self._load_player(video_id, player_url)
2880         jsi = JSInterpreter(jscode)
2881
2882         if func_code:
2883             return jsi, player_id, func_code
2884
2885         func_name = self._extract_n_function_name(jscode)
2886
2887         # For redundancy
2888         func_code = self._search_regex(
2889             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
2890                      # NB: The end of the regex is intentionally kept strict
2891                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
2892             jscode, 'nsig function', group=('var', 'code'), default=None)
2893         if func_code:
2894             func_code = ([func_code[0]], func_code[1])
2895         else:
2896             self.write_debug('Extracting nsig function with jsinterp')
2897             func_code = jsi.extract_function_code(func_name)
2898
2899         self.cache.store('youtube-nsig', player_id, func_code)
2900         return jsi, player_id, func_code
2901
2902     def _extract_n_function_from_code(self, jsi, func_code):
2903         func = jsi.extract_function_from_code(*func_code)
2904
2905         def extract_nsig(s):
2906             try:
2907                 ret = func([s])
2908             except JSInterpreter.Exception:
2909                 raise
2910             except Exception as e:
2911                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
2912
2913             if ret.startswith('enhanced_except_'):
2914                 raise JSInterpreter.Exception('Signature function returned an exception')
2915             return ret
2916
2917         return extract_nsig
2918
2919     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2920         """
2921         Extract signatureTimestamp (sts)
2922         Required to tell API what sig/player version is in use.
2923         """
2924         sts = None
2925         if isinstance(ytcfg, dict):
2926             sts = int_or_none(ytcfg.get('STS'))
2927
2928         if not sts:
2929             # Attempt to extract from player
2930             if player_url is None:
2931                 error_msg = 'Cannot extract signature timestamp without player_url.'
2932                 if fatal:
2933                     raise ExtractorError(error_msg)
2934                 self.report_warning(error_msg)
2935                 return
2936             code = self._load_player(video_id, player_url, fatal=fatal)
2937             if code:
2938                 sts = int_or_none(self._search_regex(
2939                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2940                     'JS player signature timestamp', group='sts', fatal=fatal))
2941         return sts
2942
2943     def _mark_watched(self, video_id, player_responses):
2944         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2945             label = 'fully ' if is_full else ''
2946             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2947                             expected_type=url_or_none)
2948             if not url:
2949                 self.report_warning(f'Unable to mark {label}watched')
2950                 return
2951             parsed_url = urllib.parse.urlparse(url)
2952             qs = urllib.parse.parse_qs(parsed_url.query)
2953
2954             # cpn generation algorithm is reverse engineered from base.js.
2955             # In fact it works even with dummy cpn.
2956             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2957             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2958
2959             # # more consistent results setting it to right before the end
2960             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2961
2962             qs.update({
2963                 'ver': ['2'],
2964                 'cpn': [cpn],
2965                 'cmt': video_length,
2966                 'el': 'detailpage',  # otherwise defaults to "shorts"
2967             })
2968
2969             if is_full:
2970                 # these seem to mark watchtime "history" in the real world
2971                 # they're required, so send in a single value
2972                 qs.update({
2973                     'st': 0,
2974                     'et': video_length,
2975                 })
2976
2977             url = urllib.parse.urlunparse(
2978                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2979
2980             self._download_webpage(
2981                 url, video_id, f'Marking {label}watched',
2982                 'Unable to mark watched', fatal=False)
2983
2984     @classmethod
2985     def _extract_from_webpage(cls, url, webpage):
2986         # Invidious Instances
2987         # https://github.com/yt-dlp/yt-dlp/issues/195
2988         # https://github.com/iv-org/invidious/pull/1730
2989         mobj = re.search(
2990             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
2991             webpage)
2992         if mobj:
2993             yield cls.url_result(mobj.group('url'), cls)
2994             raise cls.StopExtraction()
2995
2996         yield from super()._extract_from_webpage(url, webpage)
2997
2998         # lazyYT YouTube embed
2999         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
3000             yield cls.url_result(unescapeHTML(id_), cls, id_)
3001
3002         # Wordpress "YouTube Video Importer" plugin
3003         for m in re.findall(r'''(?x)<div[^>]+
3004                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
3005                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
3006             yield cls.url_result(m[-1], cls, m[-1])
3007
3008     @classmethod
3009     def extract_id(cls, url):
3010         video_id = cls.get_temp_id(url)
3011         if not video_id:
3012             raise ExtractorError(f'Invalid URL: {url}')
3013         return video_id
3014
3015     def _extract_chapters_from_json(self, data, duration):
3016         chapter_list = traverse_obj(
3017             data, (
3018                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3019                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3020             ), expected_type=list)
3021
3022         return self._extract_chapters(
3023             chapter_list,
3024             chapter_time=lambda chapter: float_or_none(
3025                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3026             chapter_title=lambda chapter: traverse_obj(
3027                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3028             duration=duration)
3029
3030     def _extract_chapters_from_engagement_panel(self, data, duration):
3031         content_list = traverse_obj(
3032             data,
3033             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3034             expected_type=list, default=[])
3035         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3036         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3037
3038         return next(filter(None, (
3039             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3040                                    chapter_time, chapter_title, duration)
3041             for contents in content_list)), [])
3042
3043     def _extract_chapters_from_description(self, description, duration):
3044         duration_re = r'(?:\d+:)?\d{1,2}:\d{2}'
3045         sep_re = r'(?m)^\s*(%s)\b\W*\s(%s)\s*$'
3046         return self._extract_chapters(
3047             re.findall(sep_re % (duration_re, r'.+?'), description or ''),
3048             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
3049             duration=duration, strict=False) or self._extract_chapters(
3050             re.findall(sep_re % (r'.+?', duration_re), description or ''),
3051             chapter_time=lambda x: parse_duration(x[1]), chapter_title=lambda x: x[0],
3052             duration=duration, strict=False)
3053
3054     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
3055         if not duration:
3056             return
3057         chapter_list = [{
3058             'start_time': chapter_time(chapter),
3059             'title': chapter_title(chapter),
3060         } for chapter in chapter_list or []]
3061         if not strict:
3062             chapter_list.sort(key=lambda c: c['start_time'] or 0)
3063
3064         chapters = [{'start_time': 0}]
3065         for idx, chapter in enumerate(chapter_list):
3066             if chapter['start_time'] is None:
3067                 self.report_warning(f'Incomplete chapter {idx}')
3068             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
3069                 chapters.append(chapter)
3070             elif chapter not in chapters:
3071                 self.report_warning(
3072                     f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
3073         return chapters[1:]
3074
3075     def _extract_comment(self, comment_renderer, parent=None):
3076         comment_id = comment_renderer.get('commentId')
3077         if not comment_id:
3078             return
3079
3080         text = self._get_text(comment_renderer, 'contentText')
3081
3082         # Timestamp is an estimate calculated from the current time and time_text
3083         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3084         timestamp = self._parse_time_text(time_text)
3085
3086         author = self._get_text(comment_renderer, 'authorText')
3087         author_id = try_get(comment_renderer,
3088                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3089
3090         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3091                                                        lambda x: x['likeCount']), str)) or 0
3092         author_thumbnail = try_get(comment_renderer,
3093                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3094
3095         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3096         is_favorited = 'creatorHeart' in (try_get(
3097             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3098         return {
3099             'id': comment_id,
3100             'text': text,
3101             'timestamp': timestamp,
3102             'time_text': time_text,
3103             'like_count': votes,
3104             'is_favorited': is_favorited,
3105             'author': author,
3106             'author_id': author_id,
3107             'author_thumbnail': author_thumbnail,
3108             'author_is_uploader': author_is_uploader,
3109             'parent': parent or 'root'
3110         }
3111
3112     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3113
3114         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3115
3116         def extract_header(contents):
3117             _continuation = None
3118             for content in contents:
3119                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3120                 expected_comment_count = self._get_count(
3121                     comments_header_renderer, 'countText', 'commentsCount')
3122
3123                 if expected_comment_count:
3124                     tracker['est_total'] = expected_comment_count
3125                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3126                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3127
3128                 sort_menu_item = try_get(
3129                     comments_header_renderer,
3130                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3131                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3132
3133                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3134                 if not _continuation:
3135                     continue
3136
3137                 sort_text = str_or_none(sort_menu_item.get('title'))
3138                 if not sort_text:
3139                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3140                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3141                 break
3142             return _continuation
3143
3144         def extract_thread(contents):
3145             if not parent:
3146                 tracker['current_page_thread'] = 0
3147             for content in contents:
3148                 if not parent and tracker['total_parent_comments'] >= max_parents:
3149                     yield
3150                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3151                 comment_renderer = get_first(
3152                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3153                     expected_type=dict, default={})
3154
3155                 comment = self._extract_comment(comment_renderer, parent)
3156                 if not comment:
3157                     continue
3158
3159                 tracker['running_total'] += 1
3160                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3161                 yield comment
3162
3163                 # Attempt to get the replies
3164                 comment_replies_renderer = try_get(
3165                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3166
3167                 if comment_replies_renderer:
3168                     tracker['current_page_thread'] += 1
3169                     comment_entries_iter = self._comment_entries(
3170                         comment_replies_renderer, ytcfg, video_id,
3171                         parent=comment.get('id'), tracker=tracker)
3172                     yield from itertools.islice(comment_entries_iter, min(
3173                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3174
3175         # Keeps track of counts across recursive calls
3176         if not tracker:
3177             tracker = dict(
3178                 running_total=0,
3179                 est_total=0,
3180                 current_page_thread=0,
3181                 total_parent_comments=0,
3182                 total_reply_comments=0)
3183
3184         # TODO: Deprecated
3185         # YouTube comments have a max depth of 2
3186         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3187         if max_depth:
3188             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3189                                                 'Set max replies in the max-comments extractor argument instead')
3190         if max_depth == 1 and parent:
3191             return
3192
3193         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3194             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3195
3196         continuation = self._extract_continuation(root_continuation_data)
3197
3198         response = None
3199         is_forced_continuation = False
3200         is_first_continuation = parent is None
3201         if is_first_continuation and not continuation:
3202             # Sometimes you can get comments by generating the continuation yourself,
3203             # even if YouTube initially reports them being disabled - e.g. stories comments.
3204             # Note: if the comment section is actually disabled, YouTube may return a response with
3205             # required check_get_keys missing. So we will disable that check initially in this case.
3206             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3207             is_forced_continuation = True
3208
3209         for page_num in itertools.count(0):
3210             if not continuation:
3211                 break
3212             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3213             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3214             if page_num == 0:
3215                 if is_first_continuation:
3216                     note_prefix = 'Downloading comment section API JSON'
3217                 else:
3218                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3219                         tracker['current_page_thread'], comment_prog_str)
3220             else:
3221                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3222                     '       ' if parent else '', ' replies' if parent else '',
3223                     page_num, comment_prog_str)
3224
3225             response = self._extract_response(
3226                 item_id=None, query=continuation,
3227                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3228                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3229             is_forced_continuation = False
3230             continuation_contents = traverse_obj(
3231                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3232
3233             continuation = None
3234             for continuation_section in continuation_contents:
3235                 continuation_items = traverse_obj(
3236                     continuation_section,
3237                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3238                     get_all=False, expected_type=list) or []
3239                 if is_first_continuation:
3240                     continuation = extract_header(continuation_items)
3241                     is_first_continuation = False
3242                     if continuation:
3243                         break
3244                     continue
3245
3246                 for entry in extract_thread(continuation_items):
3247                     if not entry:
3248                         return
3249                     yield entry
3250                 continuation = self._extract_continuation({'contents': continuation_items})
3251                 if continuation:
3252                     break
3253
3254         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3255         if message and not parent and tracker['running_total'] == 0:
3256             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3257
3258     @staticmethod
3259     def _generate_comment_continuation(video_id):
3260         """
3261         Generates initial comment section continuation token from given video id
3262         """
3263         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3264         return base64.b64encode(token.encode()).decode()
3265
3266     def _get_comments(self, ytcfg, video_id, contents, webpage):
3267         """Entry for comment extraction"""
3268         def _real_comment_extract(contents):
3269             renderer = next((
3270                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3271                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3272             yield from self._comment_entries(renderer, ytcfg, video_id)
3273
3274         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3275         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3276
3277     @staticmethod
3278     def _get_checkok_params():
3279         return {'contentCheckOk': True, 'racyCheckOk': True}
3280
3281     @classmethod
3282     def _generate_player_context(cls, sts=None):
3283         context = {
3284             'html5Preference': 'HTML5_PREF_WANTS',
3285         }
3286         if sts is not None:
3287             context['signatureTimestamp'] = sts
3288         return {
3289             'playbackContext': {
3290                 'contentPlaybackContext': context
3291             },
3292             **cls._get_checkok_params()
3293         }
3294
3295     @staticmethod
3296     def _is_agegated(player_response):
3297         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3298             return True
3299
3300         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3301         AGE_GATE_REASONS = (
3302             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3303             'age_verification_required', 'age_check_required',  # status
3304         )
3305         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3306
3307     @staticmethod
3308     def _is_unplayable(player_response):
3309         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3310
3311     _STORY_PLAYER_PARAMS = '8AEB'
3312
3313     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3314
3315         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3316         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3317         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3318         headers = self.generate_api_headers(
3319             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3320
3321         yt_query = {
3322             'videoId': video_id,
3323         }
3324         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3325             yt_query['params'] = self._STORY_PLAYER_PARAMS
3326
3327         yt_query.update(self._generate_player_context(sts))
3328         return self._extract_response(
3329             item_id=video_id, ep='player', query=yt_query,
3330             ytcfg=player_ytcfg, headers=headers, fatal=True,
3331             default_client=client,
3332             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3333         ) or None
3334
3335     def _get_requested_clients(self, url, smuggled_data):
3336         requested_clients = []
3337         default = ['android', 'web']
3338         allowed_clients = sorted(
3339             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3340             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3341         for client in self._configuration_arg('player_client'):
3342             if client in allowed_clients:
3343                 requested_clients.append(client)
3344             elif client == 'default':
3345                 requested_clients.extend(default)
3346             elif client == 'all':
3347                 requested_clients.extend(allowed_clients)
3348             else:
3349                 self.report_warning(f'Skipping unsupported client {client}')
3350         if not requested_clients:
3351             requested_clients = default
3352
3353         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3354             requested_clients.extend(
3355                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3356
3357         return orderedSet(requested_clients)
3358
3359     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3360         initial_pr = None
3361         if webpage:
3362             initial_pr = self._search_json(
3363                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3364
3365         all_clients = set(clients)
3366         clients = clients[::-1]
3367         prs = []
3368
3369         def append_client(*client_names):
3370             """ Append the first client name that exists but not already used """
3371             for client_name in client_names:
3372                 actual_client = _split_innertube_client(client_name)[0]
3373                 if actual_client in INNERTUBE_CLIENTS:
3374                     if actual_client not in all_clients:
3375                         clients.append(client_name)
3376                         all_clients.add(actual_client)
3377                         return
3378
3379         # Android player_response does not have microFormats which are needed for
3380         # extraction of some data. So we return the initial_pr with formats
3381         # stripped out even if not requested by the user
3382         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3383         if initial_pr:
3384             pr = dict(initial_pr)
3385             pr['streamingData'] = None
3386             prs.append(pr)
3387
3388         last_error = None
3389         tried_iframe_fallback = False
3390         player_url = None
3391         while clients:
3392             client, base_client, variant = _split_innertube_client(clients.pop())
3393             player_ytcfg = master_ytcfg if client == 'web' else {}
3394             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3395                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3396
3397             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3398             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3399             if 'js' in self._configuration_arg('player_skip'):
3400                 require_js_player = False
3401                 player_url = None
3402
3403             if not player_url and not tried_iframe_fallback and require_js_player:
3404                 player_url = self._download_player_url(video_id)
3405                 tried_iframe_fallback = True
3406
3407             try:
3408                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3409                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3410             except ExtractorError as e:
3411                 if last_error:
3412                     self.report_warning(last_error)
3413                 last_error = e
3414                 continue
3415
3416             if pr:
3417                 # YouTube may return a different video player response than expected.
3418                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3419                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3420                 if pr_video_id and pr_video_id != video_id:
3421                     self.report_warning(
3422                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3423                 else:
3424                     prs.append(pr)
3425
3426             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3427             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3428                 append_client(f'{base_client}_creator')
3429             elif self._is_agegated(pr):
3430                 if variant == 'tv_embedded':
3431                     append_client(f'{base_client}_embedded')
3432                 elif not variant:
3433                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3434
3435         if last_error:
3436             if not len(prs):
3437                 raise last_error
3438             self.report_warning(last_error)
3439         return prs, player_url
3440
3441     def _needs_live_processing(self, live_status, duration):
3442         if (live_status == 'is_live' and self.get_param('live_from_start')
3443                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3444             return live_status
3445
3446     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3447         itags, stream_ids = {}, []
3448         itag_qualities, res_qualities = {}, {0: None}
3449         q = qualities([
3450             # Normally tiny is the smallest video-only formats. But
3451             # audio-only formats with unknown quality may get tagged as tiny
3452             'tiny',
3453             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3454             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3455         ])
3456         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3457
3458         for fmt in streaming_formats:
3459             if fmt.get('targetDurationSec'):
3460                 continue
3461
3462             itag = str_or_none(fmt.get('itag'))
3463             audio_track = fmt.get('audioTrack') or {}
3464             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3465             if stream_id in stream_ids:
3466                 continue
3467
3468             quality = fmt.get('quality')
3469             height = int_or_none(fmt.get('height'))
3470             if quality == 'tiny' or not quality:
3471                 quality = fmt.get('audioQuality', '').lower() or quality
3472             # The 3gp format (17) in android client has a quality of "small",
3473             # but is actually worse than other formats
3474             if itag == '17':
3475                 quality = 'tiny'
3476             if quality:
3477                 if itag:
3478                     itag_qualities[itag] = quality
3479                 if height:
3480                     res_qualities[height] = quality
3481             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3482             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3483             # number of fragment that would subsequently requested with (`&sq=N`)
3484             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3485                 continue
3486
3487             fmt_url = fmt.get('url')
3488             if not fmt_url:
3489                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3490                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3491                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3492                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3493                     continue
3494                 try:
3495                     fmt_url += '&%s=%s' % (
3496                         traverse_obj(sc, ('sp', -1)) or 'signature',
3497                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3498                     )
3499                 except ExtractorError as e:
3500                     self.report_warning('Signature extraction failed: Some formats may be missing',
3501                                         video_id=video_id, only_once=True)
3502                     self.write_debug(e, only_once=True)
3503                     continue
3504
3505             query = parse_qs(fmt_url)
3506             throttled = False
3507             if query.get('n'):
3508                 try:
3509                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3510                     fmt_url = update_url_query(fmt_url, {
3511                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3512                     })
3513                 except ExtractorError as e:
3514                     phantomjs_hint = ''
3515                     if isinstance(e, JSInterpreter.Exception):
3516                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3517                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3518                     if player_url:
3519                         self.report_warning(
3520                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3521                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3522                         self.write_debug(e, only_once=True)
3523                     else:
3524                         self.report_warning(
3525                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3526                             video_id=video_id, only_once=True)
3527                     throttled = True
3528
3529             if itag:
3530                 itags[itag] = 'https'
3531                 stream_ids.append(stream_id)
3532
3533             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3534             language_preference = (
3535                 10 if audio_track.get('audioIsDefault') and 10
3536                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3537                 else -1)
3538             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3539             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3540             # Make sure to avoid false positives with small duration differences.
3541             # E.g. __2ABJjxzNo, ySuUZEjARPY
3542             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3543             if is_damaged:
3544                 self.report_warning(
3545                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3546             dct = {
3547                 'asr': int_or_none(fmt.get('audioSampleRate')),
3548                 'filesize': int_or_none(fmt.get('contentLength')),
3549                 'format_id': itag,
3550                 'format_note': join_nonempty(
3551                     '%s%s' % (audio_track.get('displayName') or '',
3552                               ' (default)' if language_preference > 0 else ''),
3553                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3554                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3555                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3556                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3557                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3558                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3559                 'fps': int_or_none(fmt.get('fps')) or None,
3560                 'audio_channels': fmt.get('audioChannels'),
3561                 'height': height,
3562                 'quality': q(quality),
3563                 'has_drm': bool(fmt.get('drmFamilies')),
3564                 'tbr': tbr,
3565                 'url': fmt_url,
3566                 'width': int_or_none(fmt.get('width')),
3567                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3568                                           'desc' if language_preference < -1 else ''),
3569                 'language_preference': language_preference,
3570                 # Strictly de-prioritize damaged and 3gp formats
3571                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3572             }
3573             mime_mobj = re.match(
3574                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3575             if mime_mobj:
3576                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3577                 dct.update(parse_codecs(mime_mobj.group(2)))
3578             no_audio = dct.get('acodec') == 'none'
3579             no_video = dct.get('vcodec') == 'none'
3580             if no_audio:
3581                 dct['vbr'] = tbr
3582             if no_video:
3583                 dct['abr'] = tbr
3584             if no_audio or no_video:
3585                 dct['downloader_options'] = {
3586                     # Youtube throttles chunks >~10M
3587                     'http_chunk_size': 10485760,
3588                 }
3589                 if dct.get('ext'):
3590                     dct['container'] = dct['ext'] + '_dash'
3591             yield dct
3592
3593         needs_live_processing = self._needs_live_processing(live_status, duration)
3594         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3595
3596         skip_manifests = set(self._configuration_arg('skip'))
3597         if (not self.get_param('youtube_include_hls_manifest', True)
3598                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3599                 or needs_live_processing and skip_bad_formats):
3600             skip_manifests.add('hls')
3601
3602         if not self.get_param('youtube_include_dash_manifest', True):
3603             skip_manifests.add('dash')
3604         if self._configuration_arg('include_live_dash'):
3605             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3606                                                 'Use include_incomplete_formats extractor argument instead')
3607         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3608             skip_manifests.add('dash')
3609
3610         def process_manifest_format(f, proto, itag):
3611             if itag in itags:
3612                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3613                     return False
3614                 itag = f'{itag}-{proto}'
3615             if itag:
3616                 f['format_id'] = itag
3617                 itags[itag] = proto
3618
3619             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3620             if f['quality'] == -1 and f.get('height'):
3621                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3622             return True
3623
3624         subtitles = {}
3625         for sd in streaming_data:
3626             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3627             if hls_manifest_url:
3628                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3629                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3630                 subtitles = self._merge_subtitles(subs, subtitles)
3631                 for f in fmts:
3632                     if process_manifest_format(f, 'hls', self._search_regex(
3633                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3634                         yield f
3635
3636             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3637             if dash_manifest_url:
3638                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3639                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3640                 for f in formats:
3641                     if process_manifest_format(f, 'dash', f['format_id']):
3642                         f['filesize'] = int_or_none(self._search_regex(
3643                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3644                         if needs_live_processing:
3645                             f['is_from_start'] = True
3646
3647                         yield f
3648         yield subtitles
3649
3650     def _extract_storyboard(self, player_responses, duration):
3651         spec = get_first(
3652             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3653         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3654         if not base_url:
3655             return
3656         L = len(spec) - 1
3657         for i, args in enumerate(spec):
3658             args = args.split('#')
3659             counts = list(map(int_or_none, args[:5]))
3660             if len(args) != 8 or not all(counts):
3661                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3662                 continue
3663             width, height, frame_count, cols, rows = counts
3664             N, sigh = args[6:]
3665
3666             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3667             fragment_count = frame_count / (cols * rows)
3668             fragment_duration = duration / fragment_count
3669             yield {
3670                 'format_id': f'sb{i}',
3671                 'format_note': 'storyboard',
3672                 'ext': 'mhtml',
3673                 'protocol': 'mhtml',
3674                 'acodec': 'none',
3675                 'vcodec': 'none',
3676                 'url': url,
3677                 'width': width,
3678                 'height': height,
3679                 'fps': frame_count / duration,
3680                 'rows': rows,
3681                 'columns': cols,
3682                 'fragments': [{
3683                     'url': url.replace('$M', str(j)),
3684                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3685                 } for j in range(math.ceil(fragment_count))],
3686             }
3687
3688     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3689         webpage = None
3690         if 'webpage' not in self._configuration_arg('player_skip'):
3691             query = {'bpctr': '9999999999', 'has_verified': '1'}
3692             if smuggled_data.get('is_story'):
3693                 query['pp'] = self._STORY_PLAYER_PARAMS
3694             webpage = self._download_webpage(
3695                 webpage_url, video_id, fatal=False, query=query)
3696
3697         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3698
3699         player_responses, player_url = self._extract_player_responses(
3700             self._get_requested_clients(url, smuggled_data),
3701             video_id, webpage, master_ytcfg, smuggled_data)
3702
3703         return webpage, master_ytcfg, player_responses, player_url
3704
3705     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3706         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3707         is_live = get_first(video_details, 'isLive')
3708         if is_live is None:
3709             is_live = get_first(live_broadcast_details, 'isLiveNow')
3710         live_content = get_first(video_details, 'isLiveContent')
3711         is_upcoming = get_first(video_details, 'isUpcoming')
3712         post_live = get_first(video_details, 'isPostLiveDvr')
3713         live_status = ('post_live' if post_live
3714                        else 'is_live' if is_live
3715                        else 'is_upcoming' if is_upcoming
3716                        else 'was_live' if live_content
3717                        else 'not_live' if False in (is_live, live_content)
3718                        else None)
3719         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3720         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3721
3722         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3723
3724     def _real_extract(self, url):
3725         url, smuggled_data = unsmuggle_url(url, {})
3726         video_id = self._match_id(url)
3727
3728         base_url = self.http_scheme() + '//www.youtube.com/'
3729         webpage_url = base_url + 'watch?v=' + video_id
3730
3731         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3732
3733         playability_statuses = traverse_obj(
3734             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3735
3736         trailer_video_id = get_first(
3737             playability_statuses,
3738             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3739             expected_type=str)
3740         if trailer_video_id:
3741             return self.url_result(
3742                 trailer_video_id, self.ie_key(), trailer_video_id)
3743
3744         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3745                        if webpage else (lambda x: None))
3746
3747         video_details = traverse_obj(
3748             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3749         microformats = traverse_obj(
3750             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3751             expected_type=dict, default=[])
3752
3753         translated_title = self._get_text(microformats, (..., 'title'))
3754         video_title = (self._preferred_lang and translated_title
3755                        or get_first(video_details, 'title')  # primary
3756                        or translated_title
3757                        or search_meta(['og:title', 'twitter:title', 'title']))
3758         translated_description = self._get_text(microformats, (..., 'description'))
3759         original_description = get_first(video_details, 'shortDescription')
3760         video_description = (
3761             self._preferred_lang and translated_description
3762             # If original description is blank, it will be an empty string.
3763             # Do not prefer translated description in this case.
3764             or original_description if original_description is not None else translated_description)
3765
3766         multifeed_metadata_list = get_first(
3767             player_responses,
3768             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3769             expected_type=str)
3770         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3771             if self.get_param('noplaylist'):
3772                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3773             else:
3774                 entries = []
3775                 feed_ids = []
3776                 for feed in multifeed_metadata_list.split(','):
3777                     # Unquote should take place before split on comma (,) since textual
3778                     # fields may contain comma as well (see
3779                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3780                     feed_data = urllib.parse.parse_qs(
3781                         urllib.parse.unquote_plus(feed))
3782
3783                     def feed_entry(name):
3784                         return try_get(
3785                             feed_data, lambda x: x[name][0], str)
3786
3787                     feed_id = feed_entry('id')
3788                     if not feed_id:
3789                         continue
3790                     feed_title = feed_entry('title')
3791                     title = video_title
3792                     if feed_title:
3793                         title += ' (%s)' % feed_title
3794                     entries.append({
3795                         '_type': 'url_transparent',
3796                         'ie_key': 'Youtube',
3797                         'url': smuggle_url(
3798                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3799                             {'force_singlefeed': True}),
3800                         'title': title,
3801                     })
3802                     feed_ids.append(feed_id)
3803                 self.to_screen(
3804                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3805                     % (', '.join(feed_ids), video_id))
3806                 return self.playlist_result(
3807                     entries, video_id, video_title, video_description)
3808
3809         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
3810                     or int_or_none(get_first(microformats, 'lengthSeconds'))
3811                     or parse_duration(search_meta('duration')) or None)
3812
3813         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
3814             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
3815         if live_status == 'post_live':
3816             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
3817
3818         if not formats:
3819             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3820                 self.report_drm(video_id)
3821             pemr = get_first(
3822                 playability_statuses,
3823                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3824             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3825             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3826             if subreason:
3827                 if subreason == 'The uploader has not made this video available in your country.':
3828                     countries = get_first(microformats, 'availableCountries')
3829                     if not countries:
3830                         regions_allowed = search_meta('regionsAllowed')
3831                         countries = regions_allowed.split(',') if regions_allowed else None
3832                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3833                 reason += f'. {subreason}'
3834             if reason:
3835                 self.raise_no_formats(reason, expected=True)
3836
3837         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3838         if not keywords and webpage:
3839             keywords = [
3840                 unescapeHTML(m.group('content'))
3841                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3842         for keyword in keywords:
3843             if keyword.startswith('yt:stretch='):
3844                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3845                 if mobj:
3846                     # NB: float is intentional for forcing float division
3847                     w, h = (float(v) for v in mobj.groups())
3848                     if w > 0 and h > 0:
3849                         ratio = w / h
3850                         for f in formats:
3851                             if f.get('vcodec') != 'none':
3852                                 f['stretched_ratio'] = ratio
3853                         break
3854         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3855         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3856         if thumbnail_url:
3857             thumbnails.append({
3858                 'url': thumbnail_url,
3859             })
3860         original_thumbnails = thumbnails.copy()
3861
3862         # The best resolution thumbnails sometimes does not appear in the webpage
3863         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3864         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3865         thumbnail_names = [
3866             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3867             # in resolution, these are not the custom thumbnail. So de-prioritize them
3868             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3869             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3870         ]
3871         n_thumbnail_names = len(thumbnail_names)
3872         thumbnails.extend({
3873             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3874                 video_id=video_id, name=name, ext=ext,
3875                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
3876         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3877         for thumb in thumbnails:
3878             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3879             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3880         self._remove_duplicate_formats(thumbnails)
3881         self._downloader._sort_thumbnails(original_thumbnails)
3882
3883         category = get_first(microformats, 'category') or search_meta('genre')
3884         channel_id = str_or_none(
3885             get_first(video_details, 'channelId')
3886             or get_first(microformats, 'externalChannelId')
3887             or search_meta('channelId'))
3888         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3889
3890         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3891         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3892         if not duration and live_end_time and live_start_time:
3893             duration = live_end_time - live_start_time
3894
3895         needs_live_processing = self._needs_live_processing(live_status, duration)
3896
3897         def is_bad_format(fmt):
3898             if needs_live_processing and not fmt.get('is_from_start'):
3899                 return True
3900             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
3901                     and fmt.get('protocol') == 'http_dash_segments'):
3902                 return True
3903
3904         for fmt in filter(is_bad_format, formats):
3905             fmt['preference'] = (fmt.get('preference') or -1) - 10
3906             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
3907
3908         if needs_live_processing:
3909             self._prepare_live_from_start_formats(
3910                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
3911
3912         formats.extend(self._extract_storyboard(player_responses, duration))
3913
3914         # source_preference is lower for throttled/potentially damaged formats
3915         self._sort_formats(formats, (
3916             'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto'))
3917
3918         info = {
3919             'id': video_id,
3920             'title': video_title,
3921             'formats': formats,
3922             'thumbnails': thumbnails,
3923             # The best thumbnail that we are sure exists. Prevents unnecessary
3924             # URL checking if user don't care about getting the best possible thumbnail
3925             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3926             'description': video_description,
3927             'uploader': get_first(video_details, 'author'),
3928             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3929             'uploader_url': owner_profile_url,
3930             'channel_id': channel_id,
3931             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3932             'duration': duration,
3933             'view_count': int_or_none(
3934                 get_first((video_details, microformats), (..., 'viewCount'))
3935                 or search_meta('interactionCount')),
3936             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3937             'age_limit': 18 if (
3938                 get_first(microformats, 'isFamilySafe') is False
3939                 or search_meta('isFamilyFriendly') == 'false'
3940                 or search_meta('og:restrictions:age') == '18+') else 0,
3941             'webpage_url': webpage_url,
3942             'categories': [category] if category else None,
3943             'tags': keywords,
3944             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3945             'live_status': live_status,
3946             'release_timestamp': live_start_time,
3947         }
3948
3949         subtitles = {}
3950         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3951         if pctr:
3952             def get_lang_code(track):
3953                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3954                         or track.get('languageCode'))
3955
3956             # Converted into dicts to remove duplicates
3957             captions = {
3958                 get_lang_code(sub): sub
3959                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3960             translation_languages = {
3961                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3962                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3963
3964             def process_language(container, base_url, lang_code, sub_name, query):
3965                 lang_subs = container.setdefault(lang_code, [])
3966                 for fmt in self._SUBTITLE_FORMATS:
3967                     query.update({
3968                         'fmt': fmt,
3969                     })
3970                     lang_subs.append({
3971                         'ext': fmt,
3972                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3973                         'name': sub_name,
3974                     })
3975
3976             # NB: Constructing the full subtitle dictionary is slow
3977             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
3978                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
3979             for lang_code, caption_track in captions.items():
3980                 base_url = caption_track.get('baseUrl')
3981                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3982                 if not base_url:
3983                     continue
3984                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3985                 if caption_track.get('kind') != 'asr':
3986                     if not lang_code:
3987                         continue
3988                     process_language(
3989                         subtitles, base_url, lang_code, lang_name, {})
3990                     if not caption_track.get('isTranslatable'):
3991                         continue
3992                 for trans_code, trans_name in translation_languages.items():
3993                     if not trans_code:
3994                         continue
3995                     orig_trans_code = trans_code
3996                     if caption_track.get('kind') != 'asr':
3997                         if not get_translated_subs:
3998                             continue
3999                         trans_code += f'-{lang_code}'
4000                         trans_name += format_field(lang_name, None, ' from %s')
4001                     # Add an "-orig" label to the original language so that it can be distinguished.
4002                     # The subs are returned without "-orig" as well for compatibility
4003                     if lang_code == f'a-{orig_trans_code}':
4004                         process_language(
4005                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
4006                     # Setting tlang=lang returns damaged subtitles.
4007                     process_language(automatic_captions, base_url, trans_code, trans_name,
4008                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
4009
4010         info['automatic_captions'] = automatic_captions
4011         info['subtitles'] = subtitles
4012
4013         parsed_url = urllib.parse.urlparse(url)
4014         for component in [parsed_url.fragment, parsed_url.query]:
4015             query = urllib.parse.parse_qs(component)
4016             for k, v in query.items():
4017                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4018                     d_k += '_time'
4019                     if d_k not in info and k in s_ks:
4020                         info[d_k] = parse_duration(query[k][0])
4021
4022         # Youtube Music Auto-generated description
4023         if video_description:
4024             mobj = re.search(
4025                 r'''(?xs)
4026                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4027                     (?P<album>[^\n]+)
4028                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4029                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4030                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4031                     .+\nAuto-generated\ by\ YouTube\.\s*$
4032                 ''', video_description)
4033             if mobj:
4034                 release_year = mobj.group('release_year')
4035                 release_date = mobj.group('release_date')
4036                 if release_date:
4037                     release_date = release_date.replace('-', '')
4038                     if not release_year:
4039                         release_year = release_date[:4]
4040                 info.update({
4041                     'album': mobj.group('album'.strip()),
4042                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4043                     'track': mobj.group('track').strip(),
4044                     'release_date': release_date,
4045                     'release_year': int_or_none(release_year),
4046                 })
4047
4048         initial_data = None
4049         if webpage:
4050             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4051         if not initial_data:
4052             query = {'videoId': video_id}
4053             query.update(self._get_checkok_params())
4054             initial_data = self._extract_response(
4055                 item_id=video_id, ep='next', fatal=False,
4056                 ytcfg=master_ytcfg, query=query,
4057                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4058                 note='Downloading initial data API JSON')
4059
4060         info['comment_count'] = traverse_obj(initial_data, (
4061             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4062             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
4063         ), (
4064             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4065             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
4066         ), expected_type=int_or_none, get_all=False)
4067
4068         try:  # This will error if there is no livechat
4069             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4070         except (KeyError, IndexError, TypeError):
4071             pass
4072         else:
4073             info.setdefault('subtitles', {})['live_chat'] = [{
4074                 # url is needed to set cookies
4075                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4076                 'video_id': video_id,
4077                 'ext': 'json',
4078                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4079                              else 'youtube_live_chat_replay'),
4080             }]
4081
4082         if initial_data:
4083             info['chapters'] = (
4084                 self._extract_chapters_from_json(initial_data, duration)
4085                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4086                 or self._extract_chapters_from_description(video_description, duration)
4087                 or None)
4088
4089         contents = traverse_obj(
4090             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4091             expected_type=list, default=[])
4092
4093         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4094         if vpir:
4095             stl = vpir.get('superTitleLink')
4096             if stl:
4097                 stl = self._get_text(stl)
4098                 if try_get(
4099                         vpir,
4100                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4101                     info['location'] = stl
4102                 else:
4103                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4104                     if mobj:
4105                         info.update({
4106                             'series': mobj.group(1),
4107                             'season_number': int(mobj.group(2)),
4108                             'episode_number': int(mobj.group(3)),
4109                         })
4110             for tlb in (try_get(
4111                     vpir,
4112                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4113                     list) or []):
4114                 tbrs = variadic(
4115                     traverse_obj(
4116                         tlb, 'toggleButtonRenderer',
4117                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer'),
4118                         default=[]))
4119                 for tbr in tbrs:
4120                     for getter, regex in [(
4121                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4122                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4123                                 lambda x: x['accessibility'],
4124                                 lambda x: x['accessibilityData']['accessibilityData'],
4125                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4126                         label = (try_get(tbr, getter, dict) or {}).get('label')
4127                         if label:
4128                             mobj = re.match(regex, label)
4129                             if mobj:
4130                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4131                                 break
4132             sbr_tooltip = try_get(
4133                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4134             if sbr_tooltip:
4135                 like_count, dislike_count = sbr_tooltip.split(' / ')
4136                 info.update({
4137                     'like_count': str_to_int(like_count),
4138                     'dislike_count': str_to_int(dislike_count),
4139                 })
4140             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4141             if vcr:
4142                 vc = self._get_count(vcr, 'viewCount')
4143                 # Upcoming premieres with waiting count are treated as live here
4144                 if vcr.get('isLive'):
4145                     info['concurrent_view_count'] = vc
4146                 elif info.get('view_count') is None:
4147                     info['view_count'] = vc
4148
4149         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4150         if vsir:
4151             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4152             info.update({
4153                 'channel': self._get_text(vor, 'title'),
4154                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4155
4156             rows = try_get(
4157                 vsir,
4158                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4159                 list) or []
4160             multiple_songs = False
4161             for row in rows:
4162                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4163                     multiple_songs = True
4164                     break
4165             for row in rows:
4166                 mrr = row.get('metadataRowRenderer') or {}
4167                 mrr_title = mrr.get('title')
4168                 if not mrr_title:
4169                     continue
4170                 mrr_title = self._get_text(mrr, 'title')
4171                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4172                 if mrr_title == 'License':
4173                     info['license'] = mrr_contents_text
4174                 elif not multiple_songs:
4175                     if mrr_title == 'Album':
4176                         info['album'] = mrr_contents_text
4177                     elif mrr_title == 'Artist':
4178                         info['artist'] = mrr_contents_text
4179                     elif mrr_title == 'Song':
4180                         info['track'] = mrr_contents_text
4181
4182         fallbacks = {
4183             'channel': 'uploader',
4184             'channel_id': 'uploader_id',
4185             'channel_url': 'uploader_url',
4186         }
4187
4188         # The upload date for scheduled, live and past live streams / premieres in microformats
4189         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4190         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4191         upload_date = (
4192             unified_strdate(get_first(microformats, 'uploadDate'))
4193             or unified_strdate(search_meta('uploadDate')))
4194         if not upload_date or (
4195             live_status in ('not_live', None)
4196             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4197         ):
4198             upload_date = strftime_or_none(
4199                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4200         info['upload_date'] = upload_date
4201
4202         for to, frm in fallbacks.items():
4203             if not info.get(to):
4204                 info[to] = info.get(frm)
4205
4206         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4207             v = info.get(s_k)
4208             if v:
4209                 info[d_k] = v
4210
4211         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4212
4213         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4214                       or get_first(video_details, 'isPrivate', expected_type=bool))
4215
4216         info['availability'] = (
4217             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4218             else self._availability(
4219                 is_private=is_private,
4220                 needs_premium=(
4221                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4222                     or False if initial_data and is_private is not None else None),
4223                 needs_subscription=(
4224                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4225                     or False if initial_data and is_private is not None else None),
4226                 needs_auth=info['age_limit'] >= 18,
4227                 is_unlisted=None if is_private is None else (
4228                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4229                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4230
4231         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4232
4233         self.mark_watched(video_id, player_responses)
4234
4235         return info
4236
4237
4238 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4239
4240     @staticmethod
4241     def passthrough_smuggled_data(func):
4242         def _smuggle(entries, smuggled_data):
4243             for entry in entries:
4244                 # TODO: Convert URL to music.youtube instead.
4245                 # Do we need to passthrough any other smuggled_data?
4246                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
4247                 yield entry
4248
4249         @functools.wraps(func)
4250         def wrapper(self, url):
4251             url, smuggled_data = unsmuggle_url(url, {})
4252             if self.is_music_url(url):
4253                 smuggled_data['is_music_url'] = True
4254             info_dict = func(self, url, smuggled_data)
4255             if smuggled_data and info_dict.get('entries'):
4256                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
4257             return info_dict
4258         return wrapper
4259
4260     def _extract_channel_id(self, webpage):
4261         channel_id = self._html_search_meta(
4262             'channelId', webpage, 'channel id', default=None)
4263         if channel_id:
4264             return channel_id
4265         channel_url = self._html_search_meta(
4266             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
4267              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
4268              'twitter:app:url:googleplay'), webpage, 'channel url')
4269         return self._search_regex(
4270             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
4271             channel_url, 'channel id')
4272
4273     @staticmethod
4274     def _extract_basic_item_renderer(item):
4275         # Modified from _extract_grid_item_renderer
4276         known_basic_renderers = (
4277             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4278         )
4279         for key, renderer in item.items():
4280             if not isinstance(renderer, dict):
4281                 continue
4282             elif key in known_basic_renderers:
4283                 return renderer
4284             elif key.startswith('grid') and key.endswith('Renderer'):
4285                 return renderer
4286
4287     def _grid_entries(self, grid_renderer):
4288         for item in grid_renderer['items']:
4289             if not isinstance(item, dict):
4290                 continue
4291             renderer = self._extract_basic_item_renderer(item)
4292             if not isinstance(renderer, dict):
4293                 continue
4294             title = self._get_text(renderer, 'title')
4295
4296             # playlist
4297             playlist_id = renderer.get('playlistId')
4298             if playlist_id:
4299                 yield self.url_result(
4300                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4301                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4302                     video_title=title)
4303                 continue
4304             # video
4305             video_id = renderer.get('videoId')
4306             if video_id:
4307                 yield self._extract_video(renderer)
4308                 continue
4309             # channel
4310             channel_id = renderer.get('channelId')
4311             if channel_id:
4312                 yield self.url_result(
4313                     'https://www.youtube.com/channel/%s' % channel_id,
4314                     ie=YoutubeTabIE.ie_key(), video_title=title)
4315                 continue
4316             # generic endpoint URL support
4317             ep_url = urljoin('https://www.youtube.com/', try_get(
4318                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4319                 str))
4320             if ep_url:
4321                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4322                     if ie.suitable(ep_url):
4323                         yield self.url_result(
4324                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4325                         break
4326
4327     def _music_reponsive_list_entry(self, renderer):
4328         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4329         if video_id:
4330             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4331                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4332         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4333         if playlist_id:
4334             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4335             if video_id:
4336                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4337                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4338             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4339                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4340         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4341         if browse_id:
4342             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4343                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4344
4345     def _shelf_entries_from_content(self, shelf_renderer):
4346         content = shelf_renderer.get('content')
4347         if not isinstance(content, dict):
4348             return
4349         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4350         if renderer:
4351             # TODO: add support for nested playlists so each shelf is processed
4352             # as separate playlist
4353             # TODO: this includes only first N items
4354             yield from self._grid_entries(renderer)
4355         renderer = content.get('horizontalListRenderer')
4356         if renderer:
4357             # TODO
4358             pass
4359
4360     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4361         ep = try_get(
4362             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4363             str)
4364         shelf_url = urljoin('https://www.youtube.com', ep)
4365         if shelf_url:
4366             # Skipping links to another channels, note that checking for
4367             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4368             # will not work
4369             if skip_channels and '/channels?' in shelf_url:
4370                 return
4371             title = self._get_text(shelf_renderer, 'title')
4372             yield self.url_result(shelf_url, video_title=title)
4373         # Shelf may not contain shelf URL, fallback to extraction from content
4374         yield from self._shelf_entries_from_content(shelf_renderer)
4375
4376     def _playlist_entries(self, video_list_renderer):
4377         for content in video_list_renderer['contents']:
4378             if not isinstance(content, dict):
4379                 continue
4380             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4381             if not isinstance(renderer, dict):
4382                 continue
4383             video_id = renderer.get('videoId')
4384             if not video_id:
4385                 continue
4386             yield self._extract_video(renderer)
4387
4388     def _rich_entries(self, rich_grid_renderer):
4389         renderer = traverse_obj(
4390             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4391         video_id = renderer.get('videoId')
4392         if not video_id:
4393             return
4394         yield self._extract_video(renderer)
4395
4396     def _video_entry(self, video_renderer):
4397         video_id = video_renderer.get('videoId')
4398         if video_id:
4399             return self._extract_video(video_renderer)
4400
4401     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4402         url = urljoin('https://youtube.com', traverse_obj(
4403             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4404         if url:
4405             return self.url_result(
4406                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4407
4408     def _post_thread_entries(self, post_thread_renderer):
4409         post_renderer = try_get(
4410             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4411         if not post_renderer:
4412             return
4413         # video attachment
4414         video_renderer = try_get(
4415             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4416         video_id = video_renderer.get('videoId')
4417         if video_id:
4418             entry = self._extract_video(video_renderer)
4419             if entry:
4420                 yield entry
4421         # playlist attachment
4422         playlist_id = try_get(
4423             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4424         if playlist_id:
4425             yield self.url_result(
4426                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4427                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4428         # inline video links
4429         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4430         for run in runs:
4431             if not isinstance(run, dict):
4432                 continue
4433             ep_url = try_get(
4434                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4435             if not ep_url:
4436                 continue
4437             if not YoutubeIE.suitable(ep_url):
4438                 continue
4439             ep_video_id = YoutubeIE._match_id(ep_url)
4440             if video_id == ep_video_id:
4441                 continue
4442             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4443
4444     def _post_thread_continuation_entries(self, post_thread_continuation):
4445         contents = post_thread_continuation.get('contents')
4446         if not isinstance(contents, list):
4447             return
4448         for content in contents:
4449             renderer = content.get('backstagePostThreadRenderer')
4450             if isinstance(renderer, dict):
4451                 yield from self._post_thread_entries(renderer)
4452                 continue
4453             renderer = content.get('videoRenderer')
4454             if isinstance(renderer, dict):
4455                 yield self._video_entry(renderer)
4456
4457     r''' # unused
4458     def _rich_grid_entries(self, contents):
4459         for content in contents:
4460             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4461             if video_renderer:
4462                 entry = self._video_entry(video_renderer)
4463                 if entry:
4464                     yield entry
4465     '''
4466
4467     def _report_history_entries(self, renderer):
4468         for url in traverse_obj(renderer, (
4469                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4470                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4471                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4472             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4473
4474     def _extract_entries(self, parent_renderer, continuation_list):
4475         # continuation_list is modified in-place with continuation_list = [continuation_token]
4476         continuation_list[:] = [None]
4477         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4478         for content in contents:
4479             if not isinstance(content, dict):
4480                 continue
4481             is_renderer = traverse_obj(
4482                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4483                 expected_type=dict)
4484             if not is_renderer:
4485                 if content.get('richItemRenderer'):
4486                     for entry in self._rich_entries(content['richItemRenderer']):
4487                         yield entry
4488                     continuation_list[0] = self._extract_continuation(parent_renderer)
4489                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4490                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4491                     yield from self._report_history_entries(table)
4492                     continuation_list[0] = self._extract_continuation(table)
4493                 continue
4494
4495             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4496             for isr_content in isr_contents:
4497                 if not isinstance(isr_content, dict):
4498                     continue
4499
4500                 known_renderers = {
4501                     'playlistVideoListRenderer': self._playlist_entries,
4502                     'gridRenderer': self._grid_entries,
4503                     'reelShelfRenderer': self._grid_entries,
4504                     'shelfRenderer': self._shelf_entries,
4505                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4506                     'backstagePostThreadRenderer': self._post_thread_entries,
4507                     'videoRenderer': lambda x: [self._video_entry(x)],
4508                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4509                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4510                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4511                 }
4512                 for key, renderer in isr_content.items():
4513                     if key not in known_renderers:
4514                         continue
4515                     for entry in known_renderers[key](renderer):
4516                         if entry:
4517                             yield entry
4518                     continuation_list[0] = self._extract_continuation(renderer)
4519                     break
4520
4521             if not continuation_list[0]:
4522                 continuation_list[0] = self._extract_continuation(is_renderer)
4523
4524         if not continuation_list[0]:
4525             continuation_list[0] = self._extract_continuation(parent_renderer)
4526
4527     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4528         continuation_list = [None]
4529         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4530         tab_content = try_get(tab, lambda x: x['content'], dict)
4531         if not tab_content:
4532             return
4533         parent_renderer = (
4534             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4535             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4536         yield from extract_entries(parent_renderer)
4537         continuation = continuation_list[0]
4538
4539         for page_num in itertools.count(1):
4540             if not continuation:
4541                 break
4542             headers = self.generate_api_headers(
4543                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4544             response = self._extract_response(
4545                 item_id=f'{item_id} page {page_num}',
4546                 query=continuation, headers=headers, ytcfg=ytcfg,
4547                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4548
4549             if not response:
4550                 break
4551             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4552             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4553             visitor_data = self._extract_visitor_data(response) or visitor_data
4554
4555             known_renderers = {
4556                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4557                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4558                 'gridVideoRenderer': (self._grid_entries, 'items'),
4559                 'gridChannelRenderer': (self._grid_entries, 'items'),
4560                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4561                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4562                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4563                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4564                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4565                 'playlistVideoListContinuation': (self._playlist_entries, None),
4566                 'gridContinuation': (self._grid_entries, None),
4567                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4568                 'sectionListContinuation': (extract_entries, None),  # for feeds
4569             }
4570
4571             continuation_items = traverse_obj(response, (
4572                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4573                 'appendContinuationItemsAction', 'continuationItems'
4574             ), 'continuationContents', get_all=False)
4575             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4576
4577             video_items_renderer = None
4578             for key in continuation_item.keys():
4579                 if key not in known_renderers:
4580                     continue
4581                 func, parent_key = known_renderers[key]
4582                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4583                 continuation_list = [None]
4584                 yield from func(video_items_renderer)
4585                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4586
4587             if not video_items_renderer:
4588                 break
4589
4590     @staticmethod
4591     def _extract_selected_tab(tabs, fatal=True):
4592         for tab_renderer in tabs:
4593             if tab_renderer.get('selected'):
4594                 return tab_renderer
4595         if fatal:
4596             raise ExtractorError('Unable to find selected tab')
4597
4598     @staticmethod
4599     def _extract_tab_renderers(response):
4600         return traverse_obj(
4601             response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
4602
4603     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4604         playlist_id = title = description = channel_url = channel_name = channel_id = None
4605         tags = []
4606
4607         selected_tab = self._extract_selected_tab(tabs)
4608         # Deprecated - remove when layout discontinued
4609         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4610         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
4611         metadata_renderer = try_get(
4612             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4613         if metadata_renderer:
4614             channel_name = metadata_renderer.get('title')
4615             channel_url = metadata_renderer.get('channelUrl')
4616             channel_id = metadata_renderer.get('externalId')
4617         else:
4618             metadata_renderer = try_get(
4619                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4620
4621         if metadata_renderer:
4622             title = metadata_renderer.get('title')
4623             description = metadata_renderer.get('description', '')
4624             playlist_id = channel_id
4625             tags = metadata_renderer.get('keywords', '').split()
4626
4627         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4628         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4629         def _get_uncropped(url):
4630             return url_or_none((url or '').split('=')[0] + '=s0')
4631
4632         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
4633         if avatar_thumbnails:
4634             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4635             if uncropped_avatar:
4636                 avatar_thumbnails.append({
4637                     'url': uncropped_avatar,
4638                     'id': 'avatar_uncropped',
4639                     'preference': 1
4640                 })
4641
4642         channel_banners = self._extract_thumbnails(
4643             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4644         for banner in channel_banners:
4645             banner['preference'] = -10
4646
4647         if channel_banners:
4648             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4649             if uncropped_banner:
4650                 channel_banners.append({
4651                     'url': uncropped_banner,
4652                     'id': 'banner_uncropped',
4653                     'preference': -5
4654                 })
4655
4656         # Deprecated - remove when old layout is discontinued
4657         primary_thumbnails = self._extract_thumbnails(
4658             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4659
4660         playlist_thumbnails = self._extract_thumbnails(
4661             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
4662
4663         if playlist_id is None:
4664             playlist_id = item_id
4665
4666         # Deprecated - remove primary_sidebar_renderer when old layout discontinued
4667         # Playlist stats is a text runs array containing [video count, view count, last updated].
4668         # last updated or (view count and last updated) may be missing.
4669         playlist_stats = get_first(
4670             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'),))
4671         last_updated_unix = self._parse_time_text(
4672             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
4673             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
4674
4675         view_count = self._get_count(playlist_stats, 1)
4676         if view_count is None:
4677             view_count = self._get_count(playlist_header_renderer, 'viewCountText')
4678
4679         playlist_count = self._get_count(playlist_stats, 0)
4680         if playlist_count is None:
4681             playlist_count = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
4682
4683         if title is None:
4684             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4685         title += format_field(selected_tab, 'title', ' - %s')
4686         title += format_field(selected_tab, 'expandedText', ' - %s')
4687
4688         metadata = {
4689             'playlist_id': playlist_id,
4690             'playlist_title': title,
4691             'playlist_description': description,
4692             'uploader': channel_name,
4693             'uploader_id': channel_id,
4694             'uploader_url': channel_url,
4695             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
4696             'tags': tags,
4697             'view_count': view_count,
4698             'availability': self._extract_availability(data),
4699             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4700             'playlist_count': playlist_count,
4701             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4702         }
4703         if not channel_id:
4704             owner = traverse_obj(playlist_header_renderer, 'ownerText')
4705             if not owner:
4706                 # Deprecated
4707                 owner = traverse_obj(
4708                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
4709                     ('videoOwner', 'videoOwnerRenderer', 'title'))
4710             owner_text = self._get_text(owner)
4711             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
4712             metadata.update(filter_dict({
4713                 'uploader': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
4714                 'uploader_id': browse_ep.get('browseId'),
4715                 'uploader_url': urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))
4716             }))
4717
4718         metadata.update({
4719             'channel': metadata['uploader'],
4720             'channel_id': metadata['uploader_id'],
4721             'channel_url': metadata['uploader_url']})
4722         return self.playlist_result(
4723             self._entries(
4724                 selected_tab, playlist_id, ytcfg,
4725                 self._extract_account_syncid(ytcfg, data),
4726                 self._extract_visitor_data(data, ytcfg)),
4727             **metadata)
4728
4729     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4730         first_id = last_id = response = None
4731         for page_num in itertools.count(1):
4732             videos = list(self._playlist_entries(playlist))
4733             if not videos:
4734                 return
4735             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4736             if start >= len(videos):
4737                 return
4738             yield from videos[start:]
4739             first_id = first_id or videos[0]['id']
4740             last_id = videos[-1]['id']
4741             watch_endpoint = try_get(
4742                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4743             headers = self.generate_api_headers(
4744                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4745                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4746             query = {
4747                 'playlistId': playlist_id,
4748                 'videoId': watch_endpoint.get('videoId') or last_id,
4749                 'index': watch_endpoint.get('index') or len(videos),
4750                 'params': watch_endpoint.get('params') or 'OAE%3D'
4751             }
4752             response = self._extract_response(
4753                 item_id='%s page %d' % (playlist_id, page_num),
4754                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4755                 check_get_keys='contents'
4756             )
4757             playlist = try_get(
4758                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4759
4760     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4761         title = playlist.get('title') or try_get(
4762             data, lambda x: x['titleText']['simpleText'], str)
4763         playlist_id = playlist.get('playlistId') or item_id
4764
4765         # Delegating everything except mix playlists to regular tab-based playlist URL
4766         playlist_url = urljoin(url, try_get(
4767             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4768             str))
4769
4770         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4771         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4772         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4773
4774         if playlist_url and playlist_url != url and not is_known_unviewable:
4775             return self.url_result(
4776                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4777                 video_title=title)
4778
4779         return self.playlist_result(
4780             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4781             playlist_id=playlist_id, playlist_title=title)
4782
4783     def _extract_availability(self, data):
4784         """
4785         Gets the availability of a given playlist/tab.
4786         Note: Unless YouTube tells us explicitly, we do not assume it is public
4787         @param data: response
4788         """
4789         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4790         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
4791         player_header_privacy = playlist_header_renderer.get('privacy')
4792
4793         badges = self._extract_badges(sidebar_renderer)
4794
4795         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4796         privacy_setting_icon = get_first(
4797             (playlist_header_renderer, sidebar_renderer),
4798             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
4799              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
4800             expected_type=str)
4801
4802         microformats_is_unlisted = traverse_obj(
4803             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
4804
4805         return (
4806             'public' if (
4807                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4808                 or player_header_privacy == 'PUBLIC'
4809                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
4810             else self._availability(
4811                 is_private=(
4812                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4813                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
4814                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
4815                 is_unlisted=(
4816                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4817                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
4818                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
4819                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
4820                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
4821                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
4822                 needs_auth=False))
4823
4824     @staticmethod
4825     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4826         sidebar_renderer = try_get(
4827             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4828         for item in sidebar_renderer:
4829             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4830             if renderer:
4831                 return renderer
4832
4833     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4834         """
4835         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
4836         """
4837         is_playlist = bool(traverse_obj(
4838             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
4839         if not is_playlist:
4840             return
4841         headers = self.generate_api_headers(
4842             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4843             visitor_data=self._extract_visitor_data(data, ytcfg))
4844         query = {
4845             'params': 'wgYCCAA=',
4846             'browseId': f'VL{item_id}'
4847         }
4848         return self._extract_response(
4849             item_id=item_id, headers=headers, query=query,
4850             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4851             note='Redownloading playlist API JSON with unavailable videos')
4852
4853     @functools.cached_property
4854     def skip_webpage(self):
4855         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4856
4857     def _extract_webpage(self, url, item_id, fatal=True):
4858         webpage, data = None, None
4859         for retry in self.RetryManager(fatal=fatal):
4860             try:
4861                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
4862                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4863             except ExtractorError as e:
4864                 if isinstance(e.cause, network_exceptions):
4865                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4866                         retry.error = e
4867                         continue
4868                 self._error_or_warning(e, fatal=fatal)
4869                 break
4870
4871             try:
4872                 self._extract_and_report_alerts(data)
4873             except ExtractorError as e:
4874                 self._error_or_warning(e, fatal=fatal)
4875                 break
4876
4877             # Sometimes youtube returns a webpage with incomplete ytInitialData
4878             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4879             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
4880                 retry.error = ExtractorError('Incomplete yt initial data received')
4881                 continue
4882
4883         return webpage, data
4884
4885     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4886         """Use if failed to extract ytcfg (and data) from initial webpage"""
4887         if not ytcfg and self.is_authenticated:
4888             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4889             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4890                 raise ExtractorError(
4891                     f'{msg}. If you are not downloading private content, or '
4892                     'your cookies are only for the first account and channel,'
4893                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4894                     expected=True)
4895             self.report_warning(msg, only_once=True)
4896
4897     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4898         data = None
4899         if not self.skip_webpage:
4900             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4901             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4902             # Reject webpage data if redirected to home page without explicitly requesting
4903             selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
4904             if (url != 'https://www.youtube.com/feed/recommended'
4905                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4906                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4907                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4908                 if fatal:
4909                     raise ExtractorError(msg, expected=True)
4910                 self.report_warning(msg, only_once=True)
4911         if not data:
4912             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4913             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4914         return data, ytcfg
4915
4916     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4917         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4918         resolve_response = self._extract_response(
4919             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4920             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4921         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4922         for ep_key, ep in endpoints.items():
4923             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4924             if params:
4925                 return self._extract_response(
4926                     item_id=item_id, query=params, ep=ep, headers=headers,
4927                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4928                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4929         err_note = 'Failed to resolve url (does the playlist exist?)'
4930         if fatal:
4931             raise ExtractorError(err_note, expected=True)
4932         self.report_warning(err_note, item_id)
4933
4934     _SEARCH_PARAMS = None
4935
4936     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4937         data = {'query': query}
4938         if params is NO_DEFAULT:
4939             params = self._SEARCH_PARAMS
4940         if params:
4941             data['params'] = params
4942
4943         content_keys = (
4944             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4945             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4946             # ytmusic search
4947             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4948             ('continuationContents', ),
4949         )
4950         display_id = f'query "{query}"'
4951         check_get_keys = tuple({keys[0] for keys in content_keys})
4952         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4953         self._report_playlist_authcheck(ytcfg, fatal=False)
4954
4955         continuation_list = [None]
4956         search = None
4957         for page_num in itertools.count(1):
4958             data.update(continuation_list[0] or {})
4959             headers = self.generate_api_headers(
4960                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4961             search = self._extract_response(
4962                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4963                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4964             slr_contents = traverse_obj(search, *content_keys)
4965             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4966             if not continuation_list[0]:
4967                 break
4968
4969
4970 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4971     IE_DESC = 'YouTube Tabs'
4972     _VALID_URL = r'''(?x:
4973         https?://
4974             (?:\w+\.)?
4975             (?:
4976                 youtube(?:kids)?\.com|
4977                 %(invidious)s
4978             )/
4979             (?:
4980                 (?P<channel_type>channel|c|user|browse)/|
4981                 (?P<not_channel>
4982                     feed/|hashtag/|
4983                     (?:playlist|watch)\?.*?\blist=
4984                 )|
4985                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4986             )
4987             (?P<id>[^/?\#&]+)
4988     )''' % {
4989         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4990         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4991     }
4992     IE_NAME = 'youtube:tab'
4993
4994     _TESTS = [{
4995         'note': 'playlists, multipage',
4996         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4997         'playlist_mincount': 94,
4998         'info_dict': {
4999             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5000             'title': 'Igor Kleiner - Playlists',
5001             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5002             'uploader': 'Igor Kleiner',
5003             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5004             'channel': 'Igor Kleiner',
5005             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5006             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5007             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5008             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5009             'channel_follower_count': int
5010         },
5011     }, {
5012         'note': 'playlists, multipage, different order',
5013         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5014         'playlist_mincount': 94,
5015         'info_dict': {
5016             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5017             'title': 'Igor Kleiner - Playlists',
5018             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5019             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5020             'uploader': 'Igor Kleiner',
5021             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5022             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5023             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5024             'channel': 'Igor Kleiner',
5025             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5026             'channel_follower_count': int
5027         },
5028     }, {
5029         'note': 'playlists, series',
5030         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5031         'playlist_mincount': 5,
5032         'info_dict': {
5033             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5034             'title': '3Blue1Brown - Playlists',
5035             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5036             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5037             'uploader': '3Blue1Brown',
5038             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5039             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5040             'channel': '3Blue1Brown',
5041             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5042             'tags': ['Mathematics'],
5043             'channel_follower_count': int
5044         },
5045     }, {
5046         'note': 'playlists, singlepage',
5047         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5048         'playlist_mincount': 4,
5049         'info_dict': {
5050             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5051             'title': 'ThirstForScience - Playlists',
5052             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5053             'uploader': 'ThirstForScience',
5054             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5055             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5056             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5057             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5058             'tags': 'count:13',
5059             'channel': 'ThirstForScience',
5060             'channel_follower_count': int
5061         }
5062     }, {
5063         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5064         'only_matching': True,
5065     }, {
5066         'note': 'basic, single video playlist',
5067         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5068         'info_dict': {
5069             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5070             'uploader': 'Sergey M.',
5071             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5072             'title': 'youtube-dl public playlist',
5073             'description': '',
5074             'tags': [],
5075             'view_count': int,
5076             'modified_date': '20201130',
5077             'channel': 'Sergey M.',
5078             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5079             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5080             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5081             'availability': 'public',
5082         },
5083         'playlist_count': 1,
5084     }, {
5085         'note': 'empty playlist',
5086         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5087         'info_dict': {
5088             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5089             'uploader': 'Sergey M.',
5090             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5091             'title': 'youtube-dl empty playlist',
5092             'tags': [],
5093             'channel': 'Sergey M.',
5094             'description': '',
5095             'modified_date': '20160902',
5096             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5097             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5098             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5099             'availability': 'public',
5100         },
5101         'playlist_count': 0,
5102     }, {
5103         'note': 'Home tab',
5104         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5105         'info_dict': {
5106             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5107             'title': 'lex will - Home',
5108             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5109             'uploader': 'lex will',
5110             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5111             'channel': 'lex will',
5112             'tags': ['bible', 'history', 'prophesy'],
5113             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5114             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5115             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5116             'channel_follower_count': int
5117         },
5118         'playlist_mincount': 2,
5119     }, {
5120         'note': 'Videos tab',
5121         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5122         'info_dict': {
5123             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5124             'title': 'lex will - Videos',
5125             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5126             'uploader': 'lex will',
5127             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5128             'tags': ['bible', 'history', 'prophesy'],
5129             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5130             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5131             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5132             'channel': 'lex will',
5133             'channel_follower_count': int
5134         },
5135         'playlist_mincount': 975,
5136     }, {
5137         'note': 'Videos tab, sorted by popular',
5138         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5139         'info_dict': {
5140             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5141             'title': 'lex will - Videos',
5142             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5143             'uploader': 'lex will',
5144             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5145             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5146             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5147             'channel': 'lex will',
5148             'tags': ['bible', 'history', 'prophesy'],
5149             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5150             'channel_follower_count': int
5151         },
5152         'playlist_mincount': 199,
5153     }, {
5154         'note': 'Playlists tab',
5155         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5156         'info_dict': {
5157             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5158             'title': 'lex will - Playlists',
5159             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5160             'uploader': 'lex will',
5161             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5162             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5163             'channel': 'lex will',
5164             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5165             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5166             'tags': ['bible', 'history', 'prophesy'],
5167             'channel_follower_count': int
5168         },
5169         'playlist_mincount': 17,
5170     }, {
5171         'note': 'Community tab',
5172         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5173         'info_dict': {
5174             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5175             'title': 'lex will - Community',
5176             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5177             'uploader': 'lex will',
5178             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5179             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5180             'channel': 'lex will',
5181             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5182             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5183             'tags': ['bible', 'history', 'prophesy'],
5184             'channel_follower_count': int
5185         },
5186         'playlist_mincount': 18,
5187     }, {
5188         'note': 'Channels tab',
5189         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5190         'info_dict': {
5191             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5192             'title': 'lex will - Channels',
5193             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5194             'uploader': 'lex will',
5195             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5196             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5197             'channel': 'lex will',
5198             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5199             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5200             'tags': ['bible', 'history', 'prophesy'],
5201             'channel_follower_count': int
5202         },
5203         'playlist_mincount': 12,
5204     }, {
5205         'note': 'Search tab',
5206         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5207         'playlist_mincount': 40,
5208         'info_dict': {
5209             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5210             'title': '3Blue1Brown - Search - linear algebra',
5211             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5212             'uploader': '3Blue1Brown',
5213             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5214             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5215             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5216             'tags': ['Mathematics'],
5217             'channel': '3Blue1Brown',
5218             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5219             'channel_follower_count': int
5220         },
5221     }, {
5222         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5223         'only_matching': True,
5224     }, {
5225         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5226         'only_matching': True,
5227     }, {
5228         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5229         'only_matching': True,
5230     }, {
5231         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5232         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5233         'info_dict': {
5234             'title': '29C3: Not my department',
5235             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5236             'uploader': 'Christiaan008',
5237             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5238             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5239             'tags': [],
5240             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5241             'view_count': int,
5242             'modified_date': '20150605',
5243             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5244             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5245             'channel': 'Christiaan008',
5246             'availability': 'public',
5247         },
5248         'playlist_count': 96,
5249     }, {
5250         'note': 'Large playlist',
5251         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5252         'info_dict': {
5253             'title': 'Uploads from Cauchemar',
5254             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5255             'uploader': 'Cauchemar',
5256             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5257             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5258             'tags': [],
5259             'modified_date': r're:\d{8}',
5260             'channel': 'Cauchemar',
5261             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5262             'view_count': int,
5263             'description': '',
5264             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5265             'availability': 'public',
5266         },
5267         'playlist_mincount': 1123,
5268         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5269     }, {
5270         'note': 'even larger playlist, 8832 videos',
5271         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5272         'only_matching': True,
5273     }, {
5274         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5275         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5276         'info_dict': {
5277             'title': 'Uploads from Interstellar Movie',
5278             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5279             'uploader': 'Interstellar Movie',
5280             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5281             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5282             'tags': [],
5283             'view_count': int,
5284             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5285             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5286             'channel': 'Interstellar Movie',
5287             'description': '',
5288             'modified_date': r're:\d{8}',
5289             'availability': 'public',
5290         },
5291         'playlist_mincount': 21,
5292     }, {
5293         'note': 'Playlist with "show unavailable videos" button',
5294         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5295         'info_dict': {
5296             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5297             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5298             'uploader': 'Phim Siêu Nhân Nhật Bản',
5299             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5300             'view_count': int,
5301             'channel': 'Phim Siêu Nhân Nhật Bản',
5302             'tags': [],
5303             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5304             'description': '',
5305             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5306             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5307             'modified_date': r're:\d{8}',
5308             'availability': 'public',
5309         },
5310         'playlist_mincount': 200,
5311         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5312     }, {
5313         'note': 'Playlist with unavailable videos in page 7',
5314         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5315         'info_dict': {
5316             'title': 'Uploads from BlankTV',
5317             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5318             'uploader': 'BlankTV',
5319             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5320             'channel': 'BlankTV',
5321             'channel_url': 'https://www.youtube.com/c/blanktv',
5322             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5323             'view_count': int,
5324             'tags': [],
5325             'uploader_url': 'https://www.youtube.com/c/blanktv',
5326             'modified_date': r're:\d{8}',
5327             'description': '',
5328             'availability': 'public',
5329         },
5330         'playlist_mincount': 1000,
5331         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5332     }, {
5333         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5334         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5335         'info_dict': {
5336             'title': 'Data Analysis with Dr Mike Pound',
5337             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5338             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5339             'uploader': 'Computerphile',
5340             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5341             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5342             'tags': [],
5343             'view_count': int,
5344             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5345             'channel_url': 'https://www.youtube.com/user/Computerphile',
5346             'channel': 'Computerphile',
5347             'availability': 'public',
5348             'modified_date': '20190712',
5349         },
5350         'playlist_mincount': 11,
5351     }, {
5352         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5353         'only_matching': True,
5354     }, {
5355         'note': 'Playlist URL that does not actually serve a playlist',
5356         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5357         'info_dict': {
5358             'id': 'FqZTN594JQw',
5359             'ext': 'webm',
5360             'title': "Smiley's People 01 detective, Adventure Series, Action",
5361             'uploader': 'STREEM',
5362             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5363             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5364             'upload_date': '20150526',
5365             'license': 'Standard YouTube License',
5366             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5367             'categories': ['People & Blogs'],
5368             'tags': list,
5369             'view_count': int,
5370             'like_count': int,
5371         },
5372         'params': {
5373             'skip_download': True,
5374         },
5375         'skip': 'This video is not available.',
5376         'add_ie': [YoutubeIE.ie_key()],
5377     }, {
5378         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5379         'only_matching': True,
5380     }, {
5381         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5382         'only_matching': True,
5383     }, {
5384         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5385         'info_dict': {
5386             'id': 'Wq15eF5vCbI',  # This will keep changing
5387             'ext': 'mp4',
5388             'title': str,
5389             'uploader': 'Sky News',
5390             'uploader_id': 'skynews',
5391             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5392             'upload_date': r're:\d{8}',
5393             'description': str,
5394             'categories': ['News & Politics'],
5395             'tags': list,
5396             'like_count': int,
5397             'release_timestamp': int,
5398             'channel': 'Sky News',
5399             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5400             'age_limit': 0,
5401             'view_count': int,
5402             'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
5403             'playable_in_embed': True,
5404             'release_date': r're:\d+',
5405             'availability': 'public',
5406             'live_status': 'is_live',
5407             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5408             'channel_follower_count': int,
5409             'concurrent_view_count': int,
5410         },
5411         'params': {
5412             'skip_download': True,
5413         },
5414         'expected_warnings': ['Ignoring subtitle tracks found in '],
5415     }, {
5416         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5417         'info_dict': {
5418             'id': 'a48o2S1cPoo',
5419             'ext': 'mp4',
5420             'title': 'The Young Turks - Live Main Show',
5421             'uploader': 'The Young Turks',
5422             'uploader_id': 'TheYoungTurks',
5423             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5424             'upload_date': '20150715',
5425             'license': 'Standard YouTube License',
5426             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5427             'categories': ['News & Politics'],
5428             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5429             'like_count': int,
5430         },
5431         'params': {
5432             'skip_download': True,
5433         },
5434         'only_matching': True,
5435     }, {
5436         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5437         'only_matching': True,
5438     }, {
5439         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5440         'only_matching': True,
5441     }, {
5442         'note': 'A channel that is not live. Should raise error',
5443         'url': 'https://www.youtube.com/user/numberphile/live',
5444         'only_matching': True,
5445     }, {
5446         'url': 'https://www.youtube.com/feed/trending',
5447         'only_matching': True,
5448     }, {
5449         'url': 'https://www.youtube.com/feed/library',
5450         'only_matching': True,
5451     }, {
5452         'url': 'https://www.youtube.com/feed/history',
5453         'only_matching': True,
5454     }, {
5455         'url': 'https://www.youtube.com/feed/subscriptions',
5456         'only_matching': True,
5457     }, {
5458         'url': 'https://www.youtube.com/feed/watch_later',
5459         'only_matching': True,
5460     }, {
5461         'note': 'Recommended - redirects to home page.',
5462         'url': 'https://www.youtube.com/feed/recommended',
5463         'only_matching': True,
5464     }, {
5465         'note': 'inline playlist with not always working continuations',
5466         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5467         'only_matching': True,
5468     }, {
5469         'url': 'https://www.youtube.com/course',
5470         'only_matching': True,
5471     }, {
5472         'url': 'https://www.youtube.com/zsecurity',
5473         'only_matching': True,
5474     }, {
5475         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5476         'only_matching': True,
5477     }, {
5478         'url': 'https://www.youtube.com/TheYoungTurks/live',
5479         'only_matching': True,
5480     }, {
5481         'url': 'https://www.youtube.com/hashtag/cctv9',
5482         'info_dict': {
5483             'id': 'cctv9',
5484             'title': '#cctv9',
5485             'tags': [],
5486         },
5487         'playlist_mincount': 350,
5488     }, {
5489         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5490         'only_matching': True,
5491     }, {
5492         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5493         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5494         'only_matching': True
5495     }, {
5496         'note': '/browse/ should redirect to /channel/',
5497         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5498         'only_matching': True
5499     }, {
5500         'note': 'VLPL, should redirect to playlist?list=PL...',
5501         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5502         'info_dict': {
5503             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5504             'uploader': 'NoCopyrightSounds',
5505             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5506             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5507             'title': 'NCS : All Releases 💿',
5508             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5509             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5510             'modified_date': r're:\d{8}',
5511             'view_count': int,
5512             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5513             'tags': [],
5514             'channel': 'NoCopyrightSounds',
5515             'availability': 'public',
5516         },
5517         'playlist_mincount': 166,
5518         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5519     }, {
5520         'note': 'Topic, should redirect to playlist?list=UU...',
5521         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5522         'info_dict': {
5523             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5524             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5525             'title': 'Uploads from Royalty Free Music - Topic',
5526             'uploader': 'Royalty Free Music - Topic',
5527             'tags': [],
5528             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5529             'channel': 'Royalty Free Music - Topic',
5530             'view_count': int,
5531             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5532             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5533             'modified_date': r're:\d{8}',
5534             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5535             'description': '',
5536             'availability': 'public',
5537         },
5538         'expected_warnings': [
5539             'The URL does not have a videos tab',
5540             r'[Uu]navailable videos (are|will be) hidden',
5541         ],
5542         'playlist_mincount': 101,
5543     }, {
5544         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
5545         # Treat as a general feed
5546         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5547         'info_dict': {
5548             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5549             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5550             'tags': [],
5551         },
5552         'playlist_mincount': 9,
5553     }, {
5554         'note': 'Youtube music Album',
5555         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5556         'info_dict': {
5557             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5558             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5559             'tags': [],
5560             'view_count': int,
5561             'description': '',
5562             'availability': 'unlisted',
5563             'modified_date': r're:\d{8}',
5564         },
5565         'playlist_count': 50,
5566     }, {
5567         'note': 'unlisted single video playlist',
5568         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5569         'info_dict': {
5570             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5571             'uploader': 'colethedj',
5572             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5573             'title': 'yt-dlp unlisted playlist test',
5574             'availability': 'unlisted',
5575             'tags': [],
5576             'modified_date': '20220418',
5577             'channel': 'colethedj',
5578             'view_count': int,
5579             'description': '',
5580             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5581             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5582             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5583         },
5584         'playlist_count': 1,
5585     }, {
5586         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5587         'url': 'https://www.youtube.com/feed/recommended',
5588         'info_dict': {
5589             'id': 'recommended',
5590             'title': 'recommended',
5591             'tags': [],
5592         },
5593         'playlist_mincount': 50,
5594         'params': {
5595             'skip_download': True,
5596             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5597         },
5598     }, {
5599         'note': 'API Fallback: /videos tab, sorted by oldest first',
5600         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5601         'info_dict': {
5602             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5603             'title': 'Cody\'sLab - Videos',
5604             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5605             'uploader': 'Cody\'sLab',
5606             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5607             'channel': 'Cody\'sLab',
5608             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5609             'tags': [],
5610             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5611             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5612             'channel_follower_count': int
5613         },
5614         'playlist_mincount': 650,
5615         'params': {
5616             'skip_download': True,
5617             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5618         },
5619         'skip': 'Query for sorting no longer works',
5620     }, {
5621         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5622         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5623         'info_dict': {
5624             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5625             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5626             'title': 'Uploads from Royalty Free Music - Topic',
5627             'uploader': 'Royalty Free Music - Topic',
5628             'modified_date': r're:\d{8}',
5629             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5630             'description': '',
5631             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5632             'tags': [],
5633             'channel': 'Royalty Free Music - Topic',
5634             'view_count': int,
5635             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5636             'availability': 'public',
5637         },
5638         'playlist_mincount': 101,
5639         'params': {
5640             'skip_download': True,
5641             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5642         },
5643     }, {
5644         'note': 'non-standard redirect to regional channel',
5645         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5646         'only_matching': True
5647     }, {
5648         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5649         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5650         'info_dict': {
5651             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5652             'modified_date': '20220407',
5653             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5654             'tags': [],
5655             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5656             'uploader': 'pukkandan',
5657             'availability': 'unlisted',
5658             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5659             'channel': 'pukkandan',
5660             'description': 'Test for collaborative playlist',
5661             'title': 'yt-dlp test - collaborative playlist',
5662             'view_count': int,
5663             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5664         },
5665         'playlist_mincount': 2
5666     }, {
5667         'note': 'translated tab name',
5668         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5669         'info_dict': {
5670             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5671             'tags': [],
5672             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5673             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5674             'description': '',
5675             'title': 'cole-dlp-test-acc - 再生リスト',
5676             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5677             'uploader': 'cole-dlp-test-acc',
5678             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5679             'channel': 'cole-dlp-test-acc',
5680             'channel_follower_count': int,
5681         },
5682         'playlist_mincount': 1,
5683         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5684         'expected_warnings': ['Preferring "ja"'],
5685     }, {
5686         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5687         'note': 'preferred lang set with playlist with translated video titles',
5688         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5689         'info_dict': {
5690             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5691             'tags': [],
5692             'view_count': int,
5693             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5694             'uploader': 'cole-dlp-test-acc',
5695             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5696             'channel': 'cole-dlp-test-acc',
5697             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5698             'description': 'test',
5699             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5700             'title': 'dlp test playlist',
5701             'availability': 'public',
5702         },
5703         'playlist_mincount': 1,
5704         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5705         'expected_warnings': ['Preferring "ja"'],
5706     }, {
5707         # shorts audio pivot for 2GtVksBMYFM.
5708         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5709         'info_dict': {
5710             'id': 'sfv_audio_pivot',
5711             'title': 'sfv_audio_pivot',
5712             'tags': [],
5713         },
5714         'playlist_mincount': 50,
5715
5716     }, {
5717         # Channel with a real live tab (not to be mistaken with streams tab)
5718         # Do not treat like it should redirect to live stream
5719         'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
5720         'info_dict': {
5721             'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
5722             'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
5723             'tags': [],
5724         },
5725         'playlist_mincount': 20,
5726     }, {
5727         # Tab name is not the same as tab id
5728         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
5729         'info_dict': {
5730             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5731             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
5732             'tags': [],
5733         },
5734         'playlist_mincount': 8,
5735     }, {
5736         # Home tab id is literally home. Not to get mistaken with featured
5737         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
5738         'info_dict': {
5739             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5740             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
5741             'tags': [],
5742         },
5743         'playlist_mincount': 8,
5744     }, {
5745         # Should get three playlists for videos, shorts and streams tabs
5746         'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5747         'info_dict': {
5748             'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5749             'title': 'Uploads for UCK9V2B22uJYu3N7eR_BT9QA'
5750         },
5751         'playlist_count': 3,
5752     }, {
5753         # Shorts tab with channel with handle
5754         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
5755         'info_dict': {
5756             'id': 'UC0intLFzLaudFG-xAvUEO-A',
5757             'title': 'Not Just Bikes - Shorts',
5758             'tags': 'count:12',
5759             'uploader': 'Not Just Bikes',
5760             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5761             'description': 'md5:7513148b1f02b924783157d84c4ea555',
5762             'channel_follower_count': int,
5763             'uploader_id': 'UC0intLFzLaudFG-xAvUEO-A',
5764             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
5765             'uploader_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5766             'channel': 'Not Just Bikes',
5767         },
5768         'playlist_mincount': 10,
5769     }, {
5770         # Streams tab
5771         'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
5772         'info_dict': {
5773             'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5774             'title': '中村悠一 - Live',
5775             'tags': 'count:7',
5776             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5777             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5778             'uploader_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5779             'channel': '中村悠一',
5780             'uploader_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5781             'channel_follower_count': int,
5782             'uploader': '中村悠一',
5783             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
5784         },
5785         'playlist_mincount': 60,
5786     }, {
5787         # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
5788         # See test_youtube_lists
5789         'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
5790         'only_matching': True,
5791     }, {
5792         # No uploads and no UCID given. Should fail with no uploads error
5793         # See test_youtube_lists
5794         'url': 'https://www.youtube.com/news',
5795         'only_matching': True
5796     }, {
5797         # No videos tab but has a shorts tab
5798         'url': 'https://www.youtube.com/c/TKFShorts',
5799         'info_dict': {
5800             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5801             'title': 'Shorts Break - Shorts',
5802             'tags': 'count:32',
5803             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5804             'channel': 'Shorts Break',
5805             'description': 'md5:a6c234cf3d50d878ef8721e34457cd11',
5806             'uploader': 'Shorts Break',
5807             'channel_follower_count': int,
5808             'uploader_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5809             'uploader_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
5810             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
5811         },
5812         'playlist_mincount': 30,
5813     }, {
5814         # Trending Now Tab. tab id is empty
5815         'url': 'https://www.youtube.com/feed/trending',
5816         'info_dict': {
5817             'id': 'trending',
5818             'title': 'trending - Now',
5819             'tags': [],
5820         },
5821         'playlist_mincount': 30,
5822     }, {
5823         # Trending Gaming Tab. tab id is empty
5824         'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
5825         'info_dict': {
5826             'id': 'trending',
5827             'title': 'trending - Gaming',
5828             'tags': [],
5829         },
5830         'playlist_mincount': 30,
5831     }]
5832
5833     @classmethod
5834     def suitable(cls, url):
5835         return False if YoutubeIE.suitable(url) else super().suitable(url)
5836
5837     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
5838
5839     def _get_url_mobj(self, url):
5840         mobj = self._URL_RE.match(url).groupdict()
5841         mobj.update((k, '') for k, v in mobj.items() if v is None)
5842         return mobj
5843
5844     def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
5845         tab_name = (tab.get('title') or '').lower()
5846         tab_url = urljoin(base_url, traverse_obj(
5847             tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
5848
5849         tab_id = (traverse_obj(tab, 'tabIdentifier', expected_type=str)
5850                   or tab_url and self._get_url_mobj(tab_url)['tab'][1:])
5851         if tab_id:
5852             return tab_id, tab_name
5853
5854         # Fallback to tab name if we cannot get the tab id.
5855         # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
5856         # Note that in the case of translated tab name this may result in an empty string, which we don't want.
5857         self.write_debug(f'Falling back to selected tab name: {tab_name}')
5858         return {
5859             'home': 'featured',
5860             'live': 'streams',
5861         }.get(tab_name, tab_name), tab_name
5862
5863     def _has_tab(self, tabs, tab_id):
5864         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
5865
5866     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5867     def _real_extract(self, url, smuggled_data):
5868         item_id = self._match_id(url)
5869         url = urllib.parse.urlunparse(
5870             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5871         compat_opts = self.get_param('compat_opts', [])
5872
5873         mobj = self._get_url_mobj(url)
5874         pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
5875         if is_channel:
5876             if smuggled_data.get('is_music_url'):
5877                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5878                     item_id = item_id[2:]
5879                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5880                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5881                     mdata = self._extract_tab_endpoint(
5882                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5883                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5884                                         get_all=False, expected_type=str)
5885                     if not murl:
5886                         raise ExtractorError('Failed to resolve album to playlist')
5887                     return self.url_result(murl, YoutubeTabIE)
5888                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5889                     pre = f'https://www.youtube.com/channel/{item_id}'
5890
5891         original_tab_id = tab[1:]
5892         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5893             tab = '/videos'
5894
5895         url = ''.join((pre, tab, post))
5896         mobj = self._get_url_mobj(url)
5897
5898         # Handle both video/playlist URLs
5899         qs = parse_qs(url)
5900         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5901
5902         if not video_id and mobj['not_channel'].startswith('watch'):
5903             if not playlist_id:
5904                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5905                 raise ExtractorError('Unable to recognize tab page')
5906             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5907             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5908             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5909             mobj = self._get_url_mobj(url)
5910
5911         if not self._yes_playlist(playlist_id, video_id):
5912             return self.url_result(
5913                 f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
5914
5915         data, ytcfg = self._extract_data(url, item_id)
5916
5917         # YouTube may provide a non-standard redirect to the regional channel
5918         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5919         # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
5920         redirect_url = traverse_obj(
5921             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5922         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5923             redirect_url = ''.join((
5924                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5925             self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
5926             return self.url_result(redirect_url, YoutubeTabIE)
5927
5928         tab_results = []
5929         tabs = self._extract_tab_renderers(data)
5930         if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
5931             selected_tab = self._extract_selected_tab(tabs)
5932             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
5933             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
5934
5935             if not original_tab_id and selected_tab_name:
5936                 self.to_screen('Channel URLs download all uploads of the channel. '
5937                                'To download only the videos in a specific tab, pass the tab\'s URL')
5938                 if self._has_tab(tabs, 'streams'):
5939                     tab_results.append(self.url_result(''.join((pre, '/streams', post))))
5940                 if self._has_tab(tabs, 'shorts'):
5941                     tab_results.append(self.url_result(''.join((pre, '/shorts', post))))
5942                 # XXX: Members-only tab should also be extracted
5943
5944                 if not tab_results and selected_tab_id != 'videos':
5945                     # Channel does not have streams, shorts or videos tabs
5946                     if item_id[:2] != 'UC':
5947                         raise ExtractorError('This channel has no uploads', expected=True)
5948
5949                     # Topic channels don't have /videos. Use the equivalent playlist instead
5950                     pl_id = f'UU{item_id[2:]}'
5951                     pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5952                     try:
5953                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5954                     except ExtractorError:
5955                         raise ExtractorError('This channel has no uploads', expected=True)
5956                     else:
5957                         item_id, url = pl_id, pl_url
5958                         self.to_screen(
5959                             f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
5960
5961                 elif tab_results and selected_tab_id != 'videos':
5962                     # When there are shorts/live tabs but not videos tab
5963                     url, data = ''.join((pre, post)), None
5964
5965             elif (original_tab_id or 'videos') != selected_tab_id:
5966                 if original_tab_id == 'live':
5967                     # Live tab should have redirected to the video
5968                     # Except in the case the channel has an actual live tab
5969                     # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
5970                     raise UserNotLive(video_id=mobj['id'])
5971                 elif selected_tab_name:
5972                     raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
5973
5974                 # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
5975                 url = f'{pre}{post}'
5976
5977         self.write_debug(f'Final URL: {url}')
5978
5979         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5980         if 'no-youtube-unavailable-videos' not in compat_opts:
5981             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5982         self._extract_and_report_alerts(data, only_once=True)
5983
5984         tabs = self._extract_tab_renderers(data)
5985         if tabs:
5986             tab_results[:0] = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
5987             tab_results[0].update({
5988                 'extractor_key': YoutubeTabIE.ie_key(),
5989                 'extractor': YoutubeTabIE.IE_NAME,
5990                 'webpage_url': url,
5991             })
5992
5993         if len(tab_results) == 1:
5994             return tab_results[0]
5995         elif len(tab_results) > 1:
5996             return self.playlist_result(tab_results, item_id, title=f'Uploads for {item_id}')
5997
5998         playlist = traverse_obj(
5999             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
6000         if playlist:
6001             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
6002
6003         video_id = traverse_obj(
6004             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
6005         if video_id:
6006             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
6007                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
6008             return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6009
6010         raise ExtractorError('Unable to recognize tab page')
6011
6012
6013 class YoutubePlaylistIE(InfoExtractor):
6014     IE_DESC = 'YouTube playlists'
6015     _VALID_URL = r'''(?x)(?:
6016                         (?:https?://)?
6017                         (?:\w+\.)?
6018                         (?:
6019                             (?:
6020                                 youtube(?:kids)?\.com|
6021                                 %(invidious)s
6022                             )
6023                             /.*?\?.*?\blist=
6024                         )?
6025                         (?P<id>%(playlist_id)s)
6026                      )''' % {
6027         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
6028         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
6029     }
6030     IE_NAME = 'youtube:playlist'
6031     _TESTS = [{
6032         'note': 'issue #673',
6033         'url': 'PLBB231211A4F62143',
6034         'info_dict': {
6035             'title': '[OLD]Team Fortress 2 (Class-based LP)',
6036             'id': 'PLBB231211A4F62143',
6037             'uploader': 'Wickman',
6038             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6039             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
6040             'view_count': int,
6041             'uploader_url': 'https://www.youtube.com/c/WickmanVT',
6042             'modified_date': r're:\d{8}',
6043             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6044             'channel': 'Wickman',
6045             'tags': [],
6046             'channel_url': 'https://www.youtube.com/c/WickmanVT',
6047             'availability': 'public',
6048         },
6049         'playlist_mincount': 29,
6050     }, {
6051         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6052         'info_dict': {
6053             'title': 'YDL_safe_search',
6054             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6055         },
6056         'playlist_count': 2,
6057         'skip': 'This playlist is private',
6058     }, {
6059         'note': 'embedded',
6060         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6061         'playlist_count': 4,
6062         'info_dict': {
6063             'title': 'JODA15',
6064             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6065             'uploader': 'milan',
6066             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6067             'description': '',
6068             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6069             'tags': [],
6070             'modified_date': '20140919',
6071             'view_count': int,
6072             'channel': 'milan',
6073             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6074             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6075             'availability': 'public',
6076         },
6077         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
6078     }, {
6079         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6080         'playlist_mincount': 455,
6081         'info_dict': {
6082             'title': '2018 Chinese New Singles (11/6 updated)',
6083             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6084             'uploader': 'LBK',
6085             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6086             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
6087             'channel': 'LBK',
6088             'view_count': int,
6089             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
6090             'tags': [],
6091             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
6092             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6093             'modified_date': r're:\d{8}',
6094             'availability': 'public',
6095         },
6096         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6097     }, {
6098         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
6099         'only_matching': True,
6100     }, {
6101         # music album playlist
6102         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
6103         'only_matching': True,
6104     }]
6105
6106     @classmethod
6107     def suitable(cls, url):
6108         if YoutubeTabIE.suitable(url):
6109             return False
6110         from ..utils import parse_qs
6111         qs = parse_qs(url)
6112         if qs.get('v', [None])[0]:
6113             return False
6114         return super().suitable(url)
6115
6116     def _real_extract(self, url):
6117         playlist_id = self._match_id(url)
6118         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
6119         url = update_url_query(
6120             'https://www.youtube.com/playlist',
6121             parse_qs(url) or {'list': playlist_id})
6122         if is_music_url:
6123             url = smuggle_url(url, {'is_music_url': True})
6124         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6125
6126
6127 class YoutubeYtBeIE(InfoExtractor):
6128     IE_DESC = 'youtu.be'
6129     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
6130     _TESTS = [{
6131         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
6132         'info_dict': {
6133             'id': 'yeWKywCrFtk',
6134             'ext': 'mp4',
6135             'title': 'Small Scale Baler and Braiding Rugs',
6136             'uploader': 'Backus-Page House Museum',
6137             'uploader_id': 'backuspagemuseum',
6138             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
6139             'upload_date': '20161008',
6140             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
6141             'categories': ['Nonprofits & Activism'],
6142             'tags': list,
6143             'like_count': int,
6144             'age_limit': 0,
6145             'playable_in_embed': True,
6146             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
6147             'channel': 'Backus-Page House Museum',
6148             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
6149             'live_status': 'not_live',
6150             'view_count': int,
6151             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
6152             'availability': 'public',
6153             'duration': 59,
6154             'comment_count': int,
6155             'channel_follower_count': int
6156         },
6157         'params': {
6158             'noplaylist': True,
6159             'skip_download': True,
6160         },
6161     }, {
6162         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6163         'only_matching': True,
6164     }]
6165
6166     def _real_extract(self, url):
6167         mobj = self._match_valid_url(url)
6168         video_id = mobj.group('id')
6169         playlist_id = mobj.group('playlist_id')
6170         return self.url_result(
6171             update_url_query('https://www.youtube.com/watch', {
6172                 'v': video_id,
6173                 'list': playlist_id,
6174                 'feature': 'youtu.be',
6175             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6176
6177
6178 class YoutubeLivestreamEmbedIE(InfoExtractor):
6179     IE_DESC = 'YouTube livestream embeds'
6180     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6181     _TESTS = [{
6182         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6183         'only_matching': True,
6184     }]
6185
6186     def _real_extract(self, url):
6187         channel_id = self._match_id(url)
6188         return self.url_result(
6189             f'https://www.youtube.com/channel/{channel_id}/live',
6190             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6191
6192
6193 class YoutubeYtUserIE(InfoExtractor):
6194     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6195     IE_NAME = 'youtube:user'
6196     _VALID_URL = r'ytuser:(?P<id>.+)'
6197     _TESTS = [{
6198         'url': 'ytuser:phihag',
6199         'only_matching': True,
6200     }]
6201
6202     def _real_extract(self, url):
6203         user_id = self._match_id(url)
6204         return self.url_result(
6205             'https://www.youtube.com/user/%s/videos' % user_id,
6206             ie=YoutubeTabIE.ie_key(), video_id=user_id)
6207
6208
6209 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6210     IE_NAME = 'youtube:favorites'
6211     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6212     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6213     _LOGIN_REQUIRED = True
6214     _TESTS = [{
6215         'url': ':ytfav',
6216         'only_matching': True,
6217     }, {
6218         'url': ':ytfavorites',
6219         'only_matching': True,
6220     }]
6221
6222     def _real_extract(self, url):
6223         return self.url_result(
6224             'https://www.youtube.com/playlist?list=LL',
6225             ie=YoutubeTabIE.ie_key())
6226
6227
6228 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6229     IE_NAME = 'youtube:notif'
6230     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6231     _VALID_URL = r':ytnotif(?:ication)?s?'
6232     _LOGIN_REQUIRED = True
6233     _TESTS = [{
6234         'url': ':ytnotif',
6235         'only_matching': True,
6236     }, {
6237         'url': ':ytnotifications',
6238         'only_matching': True,
6239     }]
6240
6241     def _extract_notification_menu(self, response, continuation_list):
6242         notification_list = traverse_obj(
6243             response,
6244             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6245             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6246             expected_type=list) or []
6247         continuation_list[0] = None
6248         for item in notification_list:
6249             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6250             if entry:
6251                 yield entry
6252             continuation = item.get('continuationItemRenderer')
6253             if continuation:
6254                 continuation_list[0] = continuation
6255
6256     def _extract_notification_renderer(self, notification):
6257         video_id = traverse_obj(
6258             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6259         url = f'https://www.youtube.com/watch?v={video_id}'
6260         channel_id = None
6261         if not video_id:
6262             browse_ep = traverse_obj(
6263                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6264             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6265             post_id = self._search_regex(
6266                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6267                 'post id', default=None)
6268             if not channel_id or not post_id:
6269                 return
6270             # The direct /post url redirects to this in the browser
6271             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6272
6273         channel = traverse_obj(
6274             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6275             expected_type=str)
6276         notification_title = self._get_text(notification, 'shortMessage')
6277         if notification_title:
6278             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6279         # TODO: handle recommended videos
6280         title = self._search_regex(
6281             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6282             'video title', default=None)
6283         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6284                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6285                      else None)
6286         return {
6287             '_type': 'url',
6288             'url': url,
6289             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6290             'video_id': video_id,
6291             'title': title,
6292             'channel_id': channel_id,
6293             'channel': channel,
6294             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6295             'timestamp': timestamp,
6296         }
6297
6298     def _notification_menu_entries(self, ytcfg):
6299         continuation_list = [None]
6300         response = None
6301         for page in itertools.count(1):
6302             ctoken = traverse_obj(
6303                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6304             response = self._extract_response(
6305                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6306                 ep='notification/get_notification_menu', check_get_keys='actions',
6307                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6308             yield from self._extract_notification_menu(response, continuation_list)
6309             if not continuation_list[0]:
6310                 break
6311
6312     def _real_extract(self, url):
6313         display_id = 'notifications'
6314         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6315         self._report_playlist_authcheck(ytcfg)
6316         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6317
6318
6319 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6320     IE_DESC = 'YouTube search'
6321     IE_NAME = 'youtube:search'
6322     _SEARCH_KEY = 'ytsearch'
6323     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6324     _TESTS = [{
6325         'url': 'ytsearch5:youtube-dl test video',
6326         'playlist_count': 5,
6327         'info_dict': {
6328             'id': 'youtube-dl test video',
6329             'title': 'youtube-dl test video',
6330         }
6331     }]
6332
6333
6334 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6335     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6336     _SEARCH_KEY = 'ytsearchdate'
6337     IE_DESC = 'YouTube search, newest videos first'
6338     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6339     _TESTS = [{
6340         'url': 'ytsearchdate5:youtube-dl test video',
6341         'playlist_count': 5,
6342         'info_dict': {
6343             'id': 'youtube-dl test video',
6344             'title': 'youtube-dl test video',
6345         }
6346     }]
6347
6348
6349 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6350     IE_DESC = 'YouTube search URLs with sorting and filter support'
6351     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6352     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6353     _TESTS = [{
6354         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6355         'playlist_mincount': 5,
6356         'info_dict': {
6357             'id': 'youtube-dl test video',
6358             'title': 'youtube-dl test video',
6359         }
6360     }, {
6361         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6362         'playlist_mincount': 5,
6363         'info_dict': {
6364             'id': 'python',
6365             'title': 'python',
6366         }
6367     }, {
6368         'url': 'https://www.youtube.com/results?search_query=%23cats',
6369         'playlist_mincount': 1,
6370         'info_dict': {
6371             'id': '#cats',
6372             'title': '#cats',
6373             # The test suite does not have support for nested playlists
6374             # 'entries': [{
6375             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6376             #     'title': '#cats',
6377             # }],
6378         },
6379     }, {
6380         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6381         'only_matching': True,
6382     }]
6383
6384     def _real_extract(self, url):
6385         qs = parse_qs(url)
6386         query = (qs.get('search_query') or qs.get('q'))[0]
6387         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6388
6389
6390 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6391     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6392     IE_NAME = 'youtube:music:search_url'
6393     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6394     _TESTS = [{
6395         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6396         'playlist_count': 16,
6397         'info_dict': {
6398             'id': 'royalty free music',
6399             'title': 'royalty free music',
6400         }
6401     }, {
6402         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6403         'playlist_mincount': 30,
6404         'info_dict': {
6405             'id': 'royalty free music - songs',
6406             'title': 'royalty free music - songs',
6407         },
6408         'params': {'extract_flat': 'in_playlist'}
6409     }, {
6410         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6411         'playlist_mincount': 30,
6412         'info_dict': {
6413             'id': 'royalty free music - community playlists',
6414             'title': 'royalty free music - community playlists',
6415         },
6416         'params': {'extract_flat': 'in_playlist'}
6417     }]
6418
6419     _SECTIONS = {
6420         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6421         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6422         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6423         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6424         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6425         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6426     }
6427
6428     def _real_extract(self, url):
6429         qs = parse_qs(url)
6430         query = (qs.get('search_query') or qs.get('q'))[0]
6431         params = qs.get('sp', (None,))[0]
6432         if params:
6433             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6434         else:
6435             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6436             params = self._SECTIONS.get(section)
6437             if not params:
6438                 section = None
6439         title = join_nonempty(query, section, delim=' - ')
6440         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6441
6442
6443 class YoutubeFeedsInfoExtractor(InfoExtractor):
6444     """
6445     Base class for feed extractors
6446     Subclasses must re-define the _FEED_NAME property.
6447     """
6448     _LOGIN_REQUIRED = True
6449     _FEED_NAME = 'feeds'
6450
6451     def _real_initialize(self):
6452         YoutubeBaseInfoExtractor._check_login_required(self)
6453
6454     @classproperty
6455     def IE_NAME(self):
6456         return f'youtube:{self._FEED_NAME}'
6457
6458     def _real_extract(self, url):
6459         return self.url_result(
6460             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6461
6462
6463 class YoutubeWatchLaterIE(InfoExtractor):
6464     IE_NAME = 'youtube:watchlater'
6465     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6466     _VALID_URL = r':ytwatchlater'
6467     _TESTS = [{
6468         'url': ':ytwatchlater',
6469         'only_matching': True,
6470     }]
6471
6472     def _real_extract(self, url):
6473         return self.url_result(
6474             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6475
6476
6477 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6478     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6479     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6480     _FEED_NAME = 'recommended'
6481     _LOGIN_REQUIRED = False
6482     _TESTS = [{
6483         'url': ':ytrec',
6484         'only_matching': True,
6485     }, {
6486         'url': ':ytrecommended',
6487         'only_matching': True,
6488     }, {
6489         'url': 'https://youtube.com',
6490         'only_matching': True,
6491     }]
6492
6493
6494 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6495     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6496     _VALID_URL = r':ytsub(?:scription)?s?'
6497     _FEED_NAME = 'subscriptions'
6498     _TESTS = [{
6499         'url': ':ytsubs',
6500         'only_matching': True,
6501     }, {
6502         'url': ':ytsubscriptions',
6503         'only_matching': True,
6504     }]
6505
6506
6507 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6508     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6509     _VALID_URL = r':ythis(?:tory)?'
6510     _FEED_NAME = 'history'
6511     _TESTS = [{
6512         'url': ':ythistory',
6513         'only_matching': True,
6514     }]
6515
6516
6517 class YoutubeStoriesIE(InfoExtractor):
6518     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6519     IE_NAME = 'youtube:stories'
6520     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6521     _TESTS = [{
6522         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6523         'only_matching': True,
6524     }]
6525
6526     def _real_extract(self, url):
6527         playlist_id = f'RLTD{self._match_id(url)}'
6528         return self.url_result(
6529             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6530             ie=YoutubeTabIE, video_id=playlist_id)
6531
6532
6533 class YoutubeShortsAudioPivotIE(InfoExtractor):
6534     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6535     IE_NAME = 'youtube:shorts:pivot:audio'
6536     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6537     _TESTS = [{
6538         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6539         'only_matching': True,
6540     }]
6541
6542     @staticmethod
6543     def _generate_audio_pivot_params(video_id):
6544         """
6545         Generates sfv_audio_pivot browse params for this video id
6546         """
6547         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6548         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6549
6550     def _real_extract(self, url):
6551         video_id = self._match_id(url)
6552         return self.url_result(
6553             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6554             ie=YoutubeTabIE)
6555
6556
6557 class YoutubeTruncatedURLIE(InfoExtractor):
6558     IE_NAME = 'youtube:truncated_url'
6559     IE_DESC = False  # Do not list
6560     _VALID_URL = r'''(?x)
6561         (?:https?://)?
6562         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6563         (?:watch\?(?:
6564             feature=[a-z_]+|
6565             annotation_id=annotation_[^&]+|
6566             x-yt-cl=[0-9]+|
6567             hl=[^&]*|
6568             t=[0-9]+
6569         )?
6570         |
6571             attribution_link\?a=[^&]+
6572         )
6573         $
6574     '''
6575
6576     _TESTS = [{
6577         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6578         'only_matching': True,
6579     }, {
6580         'url': 'https://www.youtube.com/watch?',
6581         'only_matching': True,
6582     }, {
6583         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6584         'only_matching': True,
6585     }, {
6586         'url': 'https://www.youtube.com/watch?feature=foo',
6587         'only_matching': True,
6588     }, {
6589         'url': 'https://www.youtube.com/watch?hl=en-GB',
6590         'only_matching': True,
6591     }, {
6592         'url': 'https://www.youtube.com/watch?t=2372',
6593         'only_matching': True,
6594     }]
6595
6596     def _real_extract(self, url):
6597         raise ExtractorError(
6598             'Did you forget to quote the URL? Remember that & is a meta '
6599             'character in most shells, so you want to put the URL in quotes, '
6600             'like  youtube-dl '
6601             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6602             ' or simply  youtube-dl BaW_jenozKc  .',
6603             expected=True)
6604
6605
6606 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6607     IE_NAME = 'youtube:clip'
6608     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6609     _TESTS = [{
6610         # FIXME: Other metadata should be extracted from the clip, not from the base video
6611         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6612         'info_dict': {
6613             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6614             'ext': 'mp4',
6615             'section_start': 29.0,
6616             'section_end': 39.7,
6617             'duration': 10.7,
6618             'age_limit': 0,
6619             'availability': 'public',
6620             'categories': ['Gaming'],
6621             'channel': 'Scott The Woz',
6622             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6623             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6624             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6625             'like_count': int,
6626             'playable_in_embed': True,
6627             'tags': 'count:17',
6628             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6629             'title': 'Mobile Games on Console - Scott The Woz',
6630             'upload_date': '20210920',
6631             'uploader': 'Scott The Woz',
6632             'uploader_id': 'scottthewoz',
6633             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6634             'view_count': int,
6635             'live_status': 'not_live',
6636             'channel_follower_count': int
6637         }
6638     }]
6639
6640     def _real_extract(self, url):
6641         clip_id = self._match_id(url)
6642         _, data = self._extract_webpage(url, clip_id)
6643
6644         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6645         if not video_id:
6646             raise ExtractorError('Unable to find video ID')
6647
6648         clip_data = traverse_obj(data, (
6649             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6650             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6651             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6652             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6653
6654         return {
6655             '_type': 'url_transparent',
6656             'url': f'https://www.youtube.com/watch?v={video_id}',
6657             'ie_key': YoutubeIE.ie_key(),
6658             'id': clip_id,
6659             'section_start': int(clip_data['startTimeMs']) / 1000,
6660             'section_end': int(clip_data['endTimeMs']) / 1000,
6661         }
6662
6663
6664 class YoutubeTruncatedIDIE(InfoExtractor):
6665     IE_NAME = 'youtube:truncated_id'
6666     IE_DESC = False  # Do not list
6667     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6668
6669     _TESTS = [{
6670         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6671         'only_matching': True,
6672     }]
6673
6674     def _real_extract(self, url):
6675         video_id = self._match_id(url)
6676         raise ExtractorError(
6677             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6678             expected=True)