yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import enum
   6 import hashlib
   7 import itertools
   8 import json
   9 import math
  10 import os.path
  11 import random
  12 import re
  13 import sys
  14 import threading
  15 import time
  16 import traceback
  17 import urllib.error
  18 import urllib.parse
  19
  20 from .common import InfoExtractor, SearchInfoExtractor
  21 from .openload import PhantomJSwrapper
  22 from ..compat import functools
  23 from ..jsinterp import JSInterpreter
  24 from ..utils import (
  25     NO_DEFAULT,
  26     ExtractorError,
  27     LazyList,
  28     UserNotLive,
  29     bug_reports_message,
  30     classproperty,
  31     clean_html,
  32     datetime_from_str,
  33     dict_get,
  34     filter_dict,
  35     float_or_none,
  36     format_field,
  37     get_first,
  38     int_or_none,
  39     is_html,
  40     join_nonempty,
  41     js_to_json,
  42     mimetype2ext,
  43     network_exceptions,
  44     orderedSet,
  45     parse_codecs,
  46     parse_count,
  47     parse_duration,
  48     parse_iso8601,
  49     parse_qs,
  50     qualities,
  51     remove_start,
  52     smuggle_url,
  53     str_or_none,
  54     str_to_int,
  55     strftime_or_none,
  56     traverse_obj,
  57     try_get,
  58     unescapeHTML,
  59     unified_strdate,
  60     unified_timestamp,
  61     unsmuggle_url,
  62     update_url_query,
  63     url_or_none,
  64     urljoin,
  65     variadic,
  66 )
  67
  68 # any clients starting with _ cannot be explicitly requested by the user
  69 INNERTUBE_CLIENTS = {
  70     'web': {
  71         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  72         'INNERTUBE_CONTEXT': {
  73             'client': {
  74                 'clientName': 'WEB',
  75                 'clientVersion': '2.20220801.00.00',
  76             }
  77         },
  78         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  79     },
  80     'web_embedded': {
  81         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  82         'INNERTUBE_CONTEXT': {
  83             'client': {
  84                 'clientName': 'WEB_EMBEDDED_PLAYER',
  85                 'clientVersion': '1.20220731.00.00',
  86             },
  87         },
  88         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  89     },
  90     'web_music': {
  91         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  92         'INNERTUBE_HOST': 'music.youtube.com',
  93         'INNERTUBE_CONTEXT': {
  94             'client': {
  95                 'clientName': 'WEB_REMIX',
  96                 'clientVersion': '1.20220727.01.00',
  97             }
  98         },
  99         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 100     },
 101     'web_creator': {
 102         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 103         'INNERTUBE_CONTEXT': {
 104             'client': {
 105                 'clientName': 'WEB_CREATOR',
 106                 'clientVersion': '1.20220726.00.00',
 107             }
 108         },
 109         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 110     },
 111     'android': {
 112         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 113         'INNERTUBE_CONTEXT': {
 114             'client': {
 115                 'clientName': 'ANDROID',
 116                 'clientVersion': '17.31.35',
 117                 'androidSdkVersion': 30,
 118                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 119             }
 120         },
 121         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 122         'REQUIRE_JS_PLAYER': False
 123     },
 124     'android_embedded': {
 125         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 126         'INNERTUBE_CONTEXT': {
 127             'client': {
 128                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 129                 'clientVersion': '17.31.35',
 130                 'androidSdkVersion': 30,
 131                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 132             },
 133         },
 134         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 135         'REQUIRE_JS_PLAYER': False
 136     },
 137     'android_music': {
 138         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 139         'INNERTUBE_CONTEXT': {
 140             'client': {
 141                 'clientName': 'ANDROID_MUSIC',
 142                 'clientVersion': '5.16.51',
 143                 'androidSdkVersion': 30,
 144                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 145             }
 146         },
 147         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 148         'REQUIRE_JS_PLAYER': False
 149     },
 150     'android_creator': {
 151         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 152         'INNERTUBE_CONTEXT': {
 153             'client': {
 154                 'clientName': 'ANDROID_CREATOR',
 155                 'clientVersion': '22.30.100',
 156                 'androidSdkVersion': 30,
 157                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 158             },
 159         },
 160         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 161         'REQUIRE_JS_PLAYER': False
 162     },
 163     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 164     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 165     'ios': {
 166         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 167         'INNERTUBE_CONTEXT': {
 168             'client': {
 169                 'clientName': 'IOS',
 170                 'clientVersion': '17.33.2',
 171                 'deviceModel': 'iPhone14,3',
 172                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 173             }
 174         },
 175         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 176         'REQUIRE_JS_PLAYER': False
 177     },
 178     'ios_embedded': {
 179         'INNERTUBE_CONTEXT': {
 180             'client': {
 181                 'clientName': 'IOS_MESSAGES_EXTENSION',
 182                 'clientVersion': '17.33.2',
 183                 'deviceModel': 'iPhone14,3',
 184                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 185             },
 186         },
 187         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 188         'REQUIRE_JS_PLAYER': False
 189     },
 190     'ios_music': {
 191         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 192         'INNERTUBE_CONTEXT': {
 193             'client': {
 194                 'clientName': 'IOS_MUSIC',
 195                 'clientVersion': '5.21',
 196                 'deviceModel': 'iPhone14,3',
 197                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 198             },
 199         },
 200         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 201         'REQUIRE_JS_PLAYER': False
 202     },
 203     'ios_creator': {
 204         'INNERTUBE_CONTEXT': {
 205             'client': {
 206                 'clientName': 'IOS_CREATOR',
 207                 'clientVersion': '22.33.101',
 208                 'deviceModel': 'iPhone14,3',
 209                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 210             },
 211         },
 212         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 213         'REQUIRE_JS_PLAYER': False
 214     },
 215     # mweb has 'ultralow' formats
 216     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 217     'mweb': {
 218         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 219         'INNERTUBE_CONTEXT': {
 220             'client': {
 221                 'clientName': 'MWEB',
 222                 'clientVersion': '2.20220801.00.00',
 223             }
 224         },
 225         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 226     },
 227     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 228     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 229     'tv_embedded': {
 230         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 231         'INNERTUBE_CONTEXT': {
 232             'client': {
 233                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 234                 'clientVersion': '2.0',
 235             },
 236         },
 237         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 238     },
 239 }
 240
 241
 242 def _split_innertube_client(client_name):
 243     variant, *base = client_name.rsplit('.', 1)
 244     if base:
 245         return variant, base[0], variant
 246     base, *variant = client_name.split('_', 1)
 247     return client_name, base, variant[0] if variant else None
 248
 249
 250 def build_innertube_clients():
 251     THIRD_PARTY = {
 252         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 253     }
 254     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 255     priority = qualities(BASE_CLIENTS[::-1])
 256
 257     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 258         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 259         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 260         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 261         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 262
 263         _, base_client, variant = _split_innertube_client(client)
 264         ytcfg['priority'] = 10 * priority(base_client)
 265
 266         if not variant:
 267             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 268             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 269             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 270             embedscreen['priority'] -= 3
 271         elif variant == 'embedded':
 272             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 273             ytcfg['priority'] -= 2
 274         else:
 275             ytcfg['priority'] -= 3
 276
 277
 278 build_innertube_clients()
 279
 280
 281 class BadgeType(enum.Enum):
 282     AVAILABILITY_UNLISTED = enum.auto()
 283     AVAILABILITY_PRIVATE = enum.auto()
 284     AVAILABILITY_PUBLIC = enum.auto()
 285     AVAILABILITY_PREMIUM = enum.auto()
 286     AVAILABILITY_SUBSCRIPTION = enum.auto()
 287     LIVE_NOW = enum.auto()
 288
 289
 290 class YoutubeBaseInfoExtractor(InfoExtractor):
 291     """Provide base functions for Youtube extractors"""
 292
 293     _RESERVED_NAMES = (
 294         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 295         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 296         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 297         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 298
 299     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 300
 301     # _NETRC_MACHINE = 'youtube'
 302
 303     # If True it will raise an error if no login info is provided
 304     _LOGIN_REQUIRED = False
 305
 306     _INVIDIOUS_SITES = (
 307         # invidious-redirect websites
 308         r'(?:www\.)?redirect\.invidious\.io',
 309         r'(?:(?:www|dev)\.)?invidio\.us',
 310         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 311         r'(?:www\.)?invidious\.pussthecat\.org',
 312         r'(?:www\.)?invidious\.zee\.li',
 313         r'(?:www\.)?invidious\.ethibox\.fr',
 314         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 315         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 316         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 317         # youtube-dl invidious instances list
 318         r'(?:(?:www|no)\.)?invidiou\.sh',
 319         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 320         r'(?:www\.)?invidious\.kabi\.tk',
 321         r'(?:www\.)?invidious\.mastodon\.host',
 322         r'(?:www\.)?invidious\.zapashcanon\.fr',
 323         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 324         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 325         r'(?:www\.)?invidious\.himiko\.cloud',
 326         r'(?:www\.)?invidious\.reallyancient\.tech',
 327         r'(?:www\.)?invidious\.tube',
 328         r'(?:www\.)?invidiou\.site',
 329         r'(?:www\.)?invidious\.site',
 330         r'(?:www\.)?invidious\.xyz',
 331         r'(?:www\.)?invidious\.nixnet\.xyz',
 332         r'(?:www\.)?invidious\.048596\.xyz',
 333         r'(?:www\.)?invidious\.drycat\.fr',
 334         r'(?:www\.)?inv\.skyn3t\.in',
 335         r'(?:www\.)?tube\.poal\.co',
 336         r'(?:www\.)?tube\.connect\.cafe',
 337         r'(?:www\.)?vid\.wxzm\.sx',
 338         r'(?:www\.)?vid\.mint\.lgbt',
 339         r'(?:www\.)?vid\.puffyan\.us',
 340         r'(?:www\.)?yewtu\.be',
 341         r'(?:www\.)?yt\.elukerio\.org',
 342         r'(?:www\.)?yt\.lelux\.fi',
 343         r'(?:www\.)?invidious\.ggc-project\.de',
 344         r'(?:www\.)?yt\.maisputain\.ovh',
 345         r'(?:www\.)?ytprivate\.com',
 346         r'(?:www\.)?invidious\.13ad\.de',
 347         r'(?:www\.)?invidious\.toot\.koeln',
 348         r'(?:www\.)?invidious\.fdn\.fr',
 349         r'(?:www\.)?watch\.nettohikari\.com',
 350         r'(?:www\.)?invidious\.namazso\.eu',
 351         r'(?:www\.)?invidious\.silkky\.cloud',
 352         r'(?:www\.)?invidious\.exonip\.de',
 353         r'(?:www\.)?invidious\.riverside\.rocks',
 354         r'(?:www\.)?invidious\.blamefran\.net',
 355         r'(?:www\.)?invidious\.moomoo\.de',
 356         r'(?:www\.)?ytb\.trom\.tf',
 357         r'(?:www\.)?yt\.cyberhost\.uk',
 358         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 359         r'(?:www\.)?qklhadlycap4cnod\.onion',
 360         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 361         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 362         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 363         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 364         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 365         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 366         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 367         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 368         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 369         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 370         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 371         r'(?:www\.)?piped\.kavin\.rocks',
 372         r'(?:www\.)?piped\.silkky\.cloud',
 373         r'(?:www\.)?piped\.tokhmi\.xyz',
 374         r'(?:www\.)?piped\.moomoo\.me',
 375         r'(?:www\.)?il\.ax',
 376         r'(?:www\.)?piped\.syncpundit\.com',
 377         r'(?:www\.)?piped\.mha\.fi',
 378         r'(?:www\.)?piped\.mint\.lgbt',
 379         r'(?:www\.)?piped\.privacy\.com\.de',
 380     )
 381
 382     # extracted from account/account_menu ep
 383     # XXX: These are the supported YouTube UI and API languages,
 384     # which is slightly different from languages supported for translation in YouTube studio
 385     _SUPPORTED_LANG_CODES = [
 386         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 387         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 388         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 389         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 390         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 391         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 392     ]
 393
 394     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 395
 396     @functools.cached_property
 397     def _preferred_lang(self):
 398         """
 399         Returns a language code supported by YouTube for the user preferred language.
 400         Returns None if no preferred language set.
 401         """
 402         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 403         if not preferred_lang:
 404             return
 405         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 406             raise ExtractorError(
 407                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 408                 expected=True)
 409         elif preferred_lang != 'en':
 410             self.report_warning(
 411                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 412         return preferred_lang
 413
 414     def _initialize_consent(self):
 415         cookies = self._get_cookies('https://www.youtube.com/')
 416         if cookies.get('__Secure-3PSID'):
 417             return
 418         consent_id = None
 419         consent = cookies.get('CONSENT')
 420         if consent:
 421             if 'YES' in consent.value:
 422                 return
 423             consent_id = self._search_regex(
 424                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 425         if not consent_id:
 426             consent_id = random.randint(100, 999)
 427         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 428
 429     def _initialize_pref(self):
 430         cookies = self._get_cookies('https://www.youtube.com/')
 431         pref_cookie = cookies.get('PREF')
 432         pref = {}
 433         if pref_cookie:
 434             try:
 435                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 436             except ValueError:
 437                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 438         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 439         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 440
 441     def _real_initialize(self):
 442         self._initialize_pref()
 443         self._initialize_consent()
 444         self._check_login_required()
 445
 446     def _check_login_required(self):
 447         if self._LOGIN_REQUIRED and not self._cookies_passed:
 448             self.raise_login_required('Login details are needed to download this content', method='cookies')
 449
 450     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 451     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 452
 453     def _get_default_ytcfg(self, client='web'):
 454         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 455
 456     def _get_innertube_host(self, client='web'):
 457         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 458
 459     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 460         # try_get but with fallback to default ytcfg client values when present
 461         _func = lambda y: try_get(y, getter, expected_type)
 462         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 463
 464     def _extract_client_name(self, ytcfg, default_client='web'):
 465         return self._ytcfg_get_safe(
 466             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 467                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 468
 469     def _extract_client_version(self, ytcfg, default_client='web'):
 470         return self._ytcfg_get_safe(
 471             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 472                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 473
 474     def _select_api_hostname(self, req_api_hostname, default_client=None):
 475         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 476                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 477
 478     def _extract_api_key(self, ytcfg=None, default_client='web'):
 479         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 480
 481     def _extract_context(self, ytcfg=None, default_client='web'):
 482         context = get_first(
 483             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 484         # Enforce language and tz for extraction
 485         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 486         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 487         return context
 488
 489     _SAPISID = None
 490
 491     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 492         time_now = round(time.time())
 493         if self._SAPISID is None:
 494             yt_cookies = self._get_cookies('https://www.youtube.com')
 495             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 496             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 497             sapisid_cookie = dict_get(
 498                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 499             if sapisid_cookie and sapisid_cookie.value:
 500                 self._SAPISID = sapisid_cookie.value
 501                 self.write_debug('Extracted SAPISID cookie')
 502                 # SAPISID cookie is required if not already present
 503                 if not yt_cookies.get('SAPISID'):
 504                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 505                     self._set_cookie(
 506                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 507             else:
 508                 self._SAPISID = False
 509         if not self._SAPISID:
 510             return None
 511         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 512         sapisidhash = hashlib.sha1(
 513             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 514         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 515
 516     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 517                   note='Downloading API JSON', errnote='Unable to download API page',
 518                   context=None, api_key=None, api_hostname=None, default_client='web'):
 519
 520         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 521         data.update(query)
 522         real_headers = self.generate_api_headers(default_client=default_client)
 523         real_headers.update({'content-type': 'application/json'})
 524         if headers:
 525             real_headers.update(headers)
 526         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 527                    or api_key or self._extract_api_key(default_client=default_client))
 528         return self._download_json(
 529             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 530             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 531             data=json.dumps(data).encode('utf8'), headers=real_headers,
 532             query={'key': api_key, 'prettyPrint': 'false'})
 533
 534     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 535         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 536
 537     @staticmethod
 538     def _extract_session_index(*data):
 539         """
 540         Index of current account in account list.
 541         See: https://github.com/yt-dlp/yt-dlp/pull/519
 542         """
 543         for ytcfg in data:
 544             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 545             if session_index is not None:
 546                 return session_index
 547
 548     # Deprecated?
 549     def _extract_identity_token(self, ytcfg=None, webpage=None):
 550         if ytcfg:
 551             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 552             if token:
 553                 return token
 554         if webpage:
 555             return self._search_regex(
 556                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 557                 'identity token', default=None, fatal=False)
 558
 559     @staticmethod
 560     def _extract_account_syncid(*args):
 561         """
 562         Extract syncId required to download private playlists of secondary channels
 563         @params response and/or ytcfg
 564         """
 565         for data in args:
 566             # ytcfg includes channel_syncid if on secondary channel
 567             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 568             if delegated_sid:
 569                 return delegated_sid
 570             sync_ids = (try_get(
 571                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 572                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 573             if len(sync_ids) >= 2 and sync_ids[1]:
 574                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 575                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 576                 return sync_ids[0]
 577
 578     @staticmethod
 579     def _extract_visitor_data(*args):
 580         """
 581         Extracts visitorData from an API response or ytcfg
 582         Appears to be used to track session state
 583         """
 584         return get_first(
 585             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 586             expected_type=str)
 587
 588     @functools.cached_property
 589     def is_authenticated(self):
 590         return bool(self._generate_sapisidhash_header())
 591
 592     def extract_ytcfg(self, video_id, webpage):
 593         if not webpage:
 594             return {}
 595         return self._parse_json(
 596             self._search_regex(
 597                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 598                 default='{}'), video_id, fatal=False) or {}
 599
 600     def generate_api_headers(
 601             self, *, ytcfg=None, account_syncid=None, session_index=None,
 602             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 603
 604         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 605         headers = {
 606             'X-YouTube-Client-Name': str(
 607                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 608             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 609             'Origin': origin,
 610             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 611             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 612             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 613             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 614         }
 615         if session_index is None:
 616             session_index = self._extract_session_index(ytcfg)
 617         if account_syncid or session_index is not None:
 618             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 619
 620         auth = self._generate_sapisidhash_header(origin)
 621         if auth is not None:
 622             headers['Authorization'] = auth
 623             headers['X-Origin'] = origin
 624         return filter_dict(headers)
 625
 626     def _download_ytcfg(self, client, video_id):
 627         url = {
 628             'web': 'https://www.youtube.com',
 629             'web_music': 'https://music.youtube.com',
 630             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 631         }.get(client)
 632         if not url:
 633             return {}
 634         webpage = self._download_webpage(
 635             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 636         return self.extract_ytcfg(video_id, webpage) or {}
 637
 638     @staticmethod
 639     def _build_api_continuation_query(continuation, ctp=None):
 640         query = {
 641             'continuation': continuation
 642         }
 643         # TODO: Inconsistency with clickTrackingParams.
 644         # Currently we have a fixed ctp contained within context (from ytcfg)
 645         # and a ctp in root query for continuation.
 646         if ctp:
 647             query['clickTracking'] = {'clickTrackingParams': ctp}
 648         return query
 649
 650     @classmethod
 651     def _extract_next_continuation_data(cls, renderer):
 652         next_continuation = try_get(
 653             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 654                        lambda x: x['continuation']['reloadContinuationData']), dict)
 655         if not next_continuation:
 656             return
 657         continuation = next_continuation.get('continuation')
 658         if not continuation:
 659             return
 660         ctp = next_continuation.get('clickTrackingParams')
 661         return cls._build_api_continuation_query(continuation, ctp)
 662
 663     @classmethod
 664     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 665         if isinstance(continuation_ep, dict):
 666             continuation = try_get(
 667                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 668             if not continuation:
 669                 return
 670             ctp = continuation_ep.get('clickTrackingParams')
 671             return cls._build_api_continuation_query(continuation, ctp)
 672
 673     @classmethod
 674     def _extract_continuation(cls, renderer):
 675         next_continuation = cls._extract_next_continuation_data(renderer)
 676         if next_continuation:
 677             return next_continuation
 678
 679         return traverse_obj(renderer, (
 680             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 681             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 682         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 683
 684     @classmethod
 685     def _extract_alerts(cls, data):
 686         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 687             if not isinstance(alert_dict, dict):
 688                 continue
 689             for alert in alert_dict.values():
 690                 alert_type = alert.get('type')
 691                 if not alert_type:
 692                     continue
 693                 message = cls._get_text(alert, 'text')
 694                 if message:
 695                     yield alert_type, message
 696
 697     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 698         errors, warnings = [], []
 699         for alert_type, alert_message in alerts:
 700             if alert_type.lower() == 'error' and fatal:
 701                 errors.append([alert_type, alert_message])
 702             elif alert_message not in self._IGNORED_WARNINGS:
 703                 warnings.append([alert_type, alert_message])
 704
 705         for alert_type, alert_message in (warnings + errors[:-1]):
 706             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 707         if errors:
 708             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 709
 710     def _extract_and_report_alerts(self, data, *args, **kwargs):
 711         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 712
 713     def _extract_badges(self, renderer: dict):
 714         privacy_icon_map = {
 715             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 716             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 717             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 718         }
 719
 720         badge_style_map = {
 721             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 722             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 723             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 724         }
 725
 726         label_map = {
 727             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 728             'private': BadgeType.AVAILABILITY_PRIVATE,
 729             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 730             'live': BadgeType.LIVE_NOW,
 731             'premium': BadgeType.AVAILABILITY_PREMIUM
 732         }
 733
 734         badges = []
 735         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer'), default=[]):
 736             badge_type = (
 737                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 738                 or badge_style_map.get(traverse_obj(badge, 'style'))
 739             )
 740             if badge_type:
 741                 badges.append({'type': badge_type})
 742                 continue
 743
 744             # fallback, won't work in some languages
 745             label = traverse_obj(badge, 'label', expected_type=str, default='')
 746             for match, label_badge_type in label_map.items():
 747                 if match in label.lower():
 748                     badges.append({'type': badge_type})
 749                     continue
 750
 751         return badges
 752
 753     @staticmethod
 754     def _has_badge(badges, badge_type):
 755         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 756
 757     @staticmethod
 758     def _get_text(data, *path_list, max_runs=None):
 759         for path in path_list or [None]:
 760             if path is None:
 761                 obj = [data]
 762             else:
 763                 obj = traverse_obj(data, path, default=[])
 764                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 765                     obj = [obj]
 766             for item in obj:
 767                 text = try_get(item, lambda x: x['simpleText'], str)
 768                 if text:
 769                     return text
 770                 runs = try_get(item, lambda x: x['runs'], list) or []
 771                 if not runs and isinstance(item, list):
 772                     runs = item
 773
 774                 runs = runs[:min(len(runs), max_runs or len(runs))]
 775                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 776                 if text:
 777                     return text
 778
 779     def _get_count(self, data, *path_list):
 780         count_text = self._get_text(data, *path_list) or ''
 781         count = parse_count(count_text)
 782         if count is None:
 783             count = str_to_int(
 784                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 785         return count
 786
 787     @staticmethod
 788     def _extract_thumbnails(data, *path_list):
 789         """
 790         Extract thumbnails from thumbnails dict
 791         @param path_list: path list to level that contains 'thumbnails' key
 792         """
 793         thumbnails = []
 794         for path in path_list or [()]:
 795             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 796                 thumbnail_url = url_or_none(thumbnail.get('url'))
 797                 if not thumbnail_url:
 798                     continue
 799                 # Sometimes youtube gives a wrong thumbnail URL. See:
 800                 # https://github.com/yt-dlp/yt-dlp/issues/233
 801                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 802                 if 'maxresdefault' in thumbnail_url:
 803                     thumbnail_url = thumbnail_url.split('?')[0]
 804                 thumbnails.append({
 805                     'url': thumbnail_url,
 806                     'height': int_or_none(thumbnail.get('height')),
 807                     'width': int_or_none(thumbnail.get('width')),
 808                 })
 809         return thumbnails
 810
 811     @staticmethod
 812     def extract_relative_time(relative_time_text):
 813         """
 814         Extracts a relative time from string and converts to dt object
 815         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 816         """
 817         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 818         if mobj:
 819             start = mobj.group('start')
 820             if start:
 821                 return datetime_from_str(start)
 822             try:
 823                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 824             except ValueError:
 825                 return None
 826
 827     def _parse_time_text(self, text):
 828         if not text:
 829             return
 830         dt = self.extract_relative_time(text)
 831         timestamp = None
 832         if isinstance(dt, datetime.datetime):
 833             timestamp = calendar.timegm(dt.timetuple())
 834
 835         if timestamp is None:
 836             timestamp = (
 837                 unified_timestamp(text) or unified_timestamp(
 838                     self._search_regex(
 839                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 840                         text.lower(), 'time text', default=None)))
 841
 842         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 843             self.report_warning(
 844                 f'Cannot parse localized time text "{text}"', only_once=True)
 845         return timestamp
 846
 847     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 848                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 849                           default_client='web'):
 850         for retry in self.RetryManager():
 851             try:
 852                 response = self._call_api(
 853                     ep=ep, fatal=True, headers=headers,
 854                     video_id=item_id, query=query, note=note,
 855                     context=self._extract_context(ytcfg, default_client),
 856                     api_key=self._extract_api_key(ytcfg, default_client),
 857                     api_hostname=api_hostname, default_client=default_client)
 858             except ExtractorError as e:
 859                 if not isinstance(e.cause, network_exceptions):
 860                     return self._error_or_warning(e, fatal=fatal)
 861                 elif not isinstance(e.cause, urllib.error.HTTPError):
 862                     retry.error = e
 863                     continue
 864
 865                 first_bytes = e.cause.read(512)
 866                 if not is_html(first_bytes):
 867                     yt_error = try_get(
 868                         self._parse_json(
 869                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 870                         lambda x: x['error']['message'], str)
 871                     if yt_error:
 872                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 873                 # Downloading page may result in intermittent 5xx HTTP error
 874                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 875                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 876                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 877                 if e.cause.code not in (403, 429):
 878                     retry.error = e
 879                     continue
 880                 return self._error_or_warning(e, fatal=fatal)
 881
 882             try:
 883                 self._extract_and_report_alerts(response, only_once=True)
 884             except ExtractorError as e:
 885                 # YouTube servers may return errors we want to retry on in a 200 OK response
 886                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 887                 if 'unknown error' in e.msg.lower():
 888                     retry.error = e
 889                     continue
 890                 return self._error_or_warning(e, fatal=fatal)
 891             # Youtube sometimes sends incomplete data
 892             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 893             if not traverse_obj(response, *variadic(check_get_keys)):
 894                 retry.error = ExtractorError('Incomplete data received', expected=True)
 895                 continue
 896
 897             return response
 898
 899     @staticmethod
 900     def is_music_url(url):
 901         return re.match(r'https?://music\.youtube\.com/', url) is not None
 902
 903     def _extract_video(self, renderer):
 904         video_id = renderer.get('videoId')
 905         title = self._get_text(renderer, 'title')
 906         description = self._get_text(renderer, 'descriptionSnippet')
 907         duration = parse_duration(self._get_text(
 908             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 909         if duration is None:
 910             duration = parse_duration(self._search_regex(
 911                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 912                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 913                 video_id, default=None, group='duration'))
 914
 915         view_count = self._get_count(renderer, 'viewCountText', 'shortViewCountText')
 916         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 917         channel_id = traverse_obj(
 918             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 919             expected_type=str, get_all=False)
 920         time_text = self._get_text(renderer, 'publishedTimeText') or ''
 921         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 922         overlay_style = traverse_obj(
 923             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 924             get_all=False, expected_type=str)
 925         badges = self._extract_badges(renderer)
 926         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 927         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 928             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 929             expected_type=str)) or ''
 930         url = f'https://www.youtube.com/watch?v={video_id}'
 931         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 932             url = f'https://www.youtube.com/shorts/{video_id}'
 933
 934         live_status = (
 935             'is_upcoming' if scheduled_timestamp is not None
 936             else 'was_live' if 'streamed' in time_text.lower()
 937             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
 938             else None)
 939
 940         return {
 941             '_type': 'url',
 942             'ie_key': YoutubeIE.ie_key(),
 943             'id': video_id,
 944             'url': url,
 945             'title': title,
 946             'description': description,
 947             'duration': duration,
 948             'uploader': uploader,
 949             'channel_id': channel_id,
 950             'thumbnails': thumbnails,
 951             'timestamp': (self._parse_time_text(time_text)
 952                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
 953                           else None),
 954             'release_timestamp': scheduled_timestamp,
 955             'availability':
 956                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
 957                 else self._availability(
 958                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
 959                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
 960                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
 961                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
 962             'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count': view_count,
 963         }
 964
 965
 966 class YoutubeIE(YoutubeBaseInfoExtractor):
 967     IE_DESC = 'YouTube'
 968     _VALID_URL = r"""(?x)^
 969                      (
 970                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 971                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 972                             (?:www\.)?deturl\.com/www\.youtube\.com|
 973                             (?:www\.)?pwnyoutube\.com|
 974                             (?:www\.)?hooktube\.com|
 975                             (?:www\.)?yourepeat\.com|
 976                             tube\.majestyc\.net|
 977                             %(invidious)s|
 978                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 979                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 980                          (?:                                                  # the various things that can precede the ID:
 981                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 982                              |(?:                                             # or the v= param in all its forms
 983                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 984                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 985                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 986                                  v=
 987                              )
 988                          ))
 989                          |(?:
 990                             youtu\.be|                                        # just youtu.be/xxxx
 991                             vid\.plus|                                        # or vid.plus/xxxx
 992                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 993                             %(invidious)s
 994                          )/
 995                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 996                          )
 997                      )?                                                       # all until now is optional -> you can pass the naked ID
 998                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 999                      (?(1).+)?                                                # if we found the ID, everything can follow
1000                      (?:\#|$)""" % {
1001         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1002     }
1003     _EMBED_REGEX = [
1004         r'''(?x)
1005             (?:
1006                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1007                 data-video-url=|
1008                 <embed[^>]+?src=|
1009                 embedSWF\(?:\s*|
1010                 <object[^>]+data=|
1011                 new\s+SWFObject\(
1012             )
1013             (["\'])
1014                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1015                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1016             \1''',
1017         # https://wordpress.org/plugins/lazy-load-for-videos/
1018         r'''(?xs)
1019             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1020             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1021     ]
1022
1023     _PLAYER_INFO_RE = (
1024         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1025         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1026         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1027     )
1028     _formats = {
1029         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1030         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1031         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1032         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1033         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1034         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1035         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1036         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1037         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1038         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1039         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1040         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1041         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1042         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1043         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1044         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1045         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1046         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1047
1048
1049         # 3D videos
1050         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1051         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1052         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1053         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1054         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1055         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1056         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1057
1058         # Apple HTTP Live Streaming
1059         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1060         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1061         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1062         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1063         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1064         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1065         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1066         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1067
1068         # DASH mp4 video
1069         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1070         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1071         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1072         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1073         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1074         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1075         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1076         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1077         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1078         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1079         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1080         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1081
1082         # Dash mp4 audio
1083         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1084         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1085         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1086         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1087         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1088         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1089         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1090
1091         # Dash webm
1092         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1093         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1094         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1095         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1096         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1097         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1098         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1099         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1100         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1101         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1102         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1103         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1104         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1105         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1106         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1107         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1108         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1109         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1110         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1111         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1112         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1113         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1114
1115         # Dash webm audio
1116         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1117         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1118
1119         # Dash webm audio with opus inside
1120         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1121         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1122         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1123
1124         # RTMP (unnamed)
1125         '_rtmp': {'protocol': 'rtmp'},
1126
1127         # av01 video only formats sometimes served with "unknown" codecs
1128         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1129         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1130         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1131         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1132         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1133         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1134         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1135         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1136     }
1137     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1138
1139     _GEO_BYPASS = False
1140
1141     IE_NAME = 'youtube'
1142     _TESTS = [
1143         {
1144             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1145             'info_dict': {
1146                 'id': 'BaW_jenozKc',
1147                 'ext': 'mp4',
1148                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1149                 'uploader': 'Philipp Hagemeister',
1150                 'uploader_id': 'phihag',
1151                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1152                 'channel': 'Philipp Hagemeister',
1153                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1154                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1155                 'upload_date': '20121002',
1156                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1157                 'categories': ['Science & Technology'],
1158                 'tags': ['youtube-dl'],
1159                 'duration': 10,
1160                 'view_count': int,
1161                 'like_count': int,
1162                 'availability': 'public',
1163                 'playable_in_embed': True,
1164                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1165                 'live_status': 'not_live',
1166                 'age_limit': 0,
1167                 'start_time': 1,
1168                 'end_time': 9,
1169                 'comment_count': int,
1170                 'channel_follower_count': int
1171             }
1172         },
1173         {
1174             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1175             'note': 'Embed-only video (#1746)',
1176             'info_dict': {
1177                 'id': 'yZIXLfi8CZQ',
1178                 'ext': 'mp4',
1179                 'upload_date': '20120608',
1180                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1181                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1182                 'uploader': 'SET India',
1183                 'uploader_id': 'setindia',
1184                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1185                 'age_limit': 18,
1186             },
1187             'skip': 'Private video',
1188         },
1189         {
1190             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1191             'note': 'Use the first video ID in the URL',
1192             'info_dict': {
1193                 'id': 'BaW_jenozKc',
1194                 'ext': 'mp4',
1195                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1196                 'uploader': 'Philipp Hagemeister',
1197                 'uploader_id': 'phihag',
1198                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1199                 'channel': 'Philipp Hagemeister',
1200                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1201                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1202                 'upload_date': '20121002',
1203                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1204                 'categories': ['Science & Technology'],
1205                 'tags': ['youtube-dl'],
1206                 'duration': 10,
1207                 'view_count': int,
1208                 'like_count': int,
1209                 'availability': 'public',
1210                 'playable_in_embed': True,
1211                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1212                 'live_status': 'not_live',
1213                 'age_limit': 0,
1214                 'comment_count': int,
1215                 'channel_follower_count': int
1216             },
1217             'params': {
1218                 'skip_download': True,
1219             },
1220         },
1221         {
1222             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1223             'note': '256k DASH audio (format 141) via DASH manifest',
1224             'info_dict': {
1225                 'id': 'a9LDPn-MO4I',
1226                 'ext': 'm4a',
1227                 'upload_date': '20121002',
1228                 'uploader_id': '8KVIDEO',
1229                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1230                 'description': '',
1231                 'uploader': '8KVIDEO',
1232                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1233             },
1234             'params': {
1235                 'youtube_include_dash_manifest': True,
1236                 'format': '141',
1237             },
1238             'skip': 'format 141 not served anymore',
1239         },
1240         # DASH manifest with encrypted signature
1241         {
1242             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1243             'info_dict': {
1244                 'id': 'IB3lcPjvWLA',
1245                 'ext': 'm4a',
1246                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1247                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1248                 'duration': 244,
1249                 'uploader': 'AfrojackVEVO',
1250                 'uploader_id': 'AfrojackVEVO',
1251                 'upload_date': '20131011',
1252                 'abr': 129.495,
1253                 'like_count': int,
1254                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1255                 'playable_in_embed': True,
1256                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1257                 'view_count': int,
1258                 'track': 'The Spark',
1259                 'live_status': 'not_live',
1260                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1261                 'channel': 'Afrojack',
1262                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1263                 'tags': 'count:19',
1264                 'availability': 'public',
1265                 'categories': ['Music'],
1266                 'age_limit': 0,
1267                 'alt_title': 'The Spark',
1268                 'channel_follower_count': int
1269             },
1270             'params': {
1271                 'youtube_include_dash_manifest': True,
1272                 'format': '141/bestaudio[ext=m4a]',
1273             },
1274         },
1275         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1276         {
1277             'note': 'Embed allowed age-gate video',
1278             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1279             'info_dict': {
1280                 'id': 'HtVdAasjOgU',
1281                 'ext': 'mp4',
1282                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1283                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1284                 'duration': 142,
1285                 'uploader': 'The Witcher',
1286                 'uploader_id': 'WitcherGame',
1287                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1288                 'upload_date': '20140605',
1289                 'age_limit': 18,
1290                 'categories': ['Gaming'],
1291                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1292                 'availability': 'needs_auth',
1293                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1294                 'like_count': int,
1295                 'channel': 'The Witcher',
1296                 'live_status': 'not_live',
1297                 'tags': 'count:17',
1298                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1299                 'playable_in_embed': True,
1300                 'view_count': int,
1301                 'channel_follower_count': int
1302             },
1303         },
1304         {
1305             'note': 'Age-gate video with embed allowed in public site',
1306             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1307             'info_dict': {
1308                 'id': 'HsUATh_Nc2U',
1309                 'ext': 'mp4',
1310                 'title': 'Godzilla 2 (Official Video)',
1311                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1312                 'upload_date': '20200408',
1313                 'uploader_id': 'FlyingKitty900',
1314                 'uploader': 'FlyingKitty',
1315                 'age_limit': 18,
1316                 'availability': 'needs_auth',
1317                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1318                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1319                 'channel': 'FlyingKitty',
1320                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1321                 'view_count': int,
1322                 'categories': ['Entertainment'],
1323                 'live_status': 'not_live',
1324                 'tags': ['Flyingkitty', 'godzilla 2'],
1325                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1326                 'like_count': int,
1327                 'duration': 177,
1328                 'playable_in_embed': True,
1329                 'channel_follower_count': int
1330             },
1331         },
1332         {
1333             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1334             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1335             'info_dict': {
1336                 'id': 'Tq92D6wQ1mg',
1337                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1338                 'ext': 'mp4',
1339                 'upload_date': '20191228',
1340                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1341                 'uploader': 'Projekt Melody',
1342                 'description': 'md5:17eccca93a786d51bc67646756894066',
1343                 'age_limit': 18,
1344                 'like_count': int,
1345                 'availability': 'needs_auth',
1346                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1347                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1348                 'view_count': int,
1349                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1350                 'channel': 'Projekt Melody',
1351                 'live_status': 'not_live',
1352                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1353                 'playable_in_embed': True,
1354                 'categories': ['Entertainment'],
1355                 'duration': 106,
1356                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1357                 'comment_count': int,
1358                 'channel_follower_count': int
1359             },
1360         },
1361         {
1362             'note': 'Non-Agegated non-embeddable video',
1363             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1364             'info_dict': {
1365                 'id': 'MeJVWBSsPAY',
1366                 'ext': 'mp4',
1367                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1368                 'uploader': 'Herr Lurik',
1369                 'uploader_id': 'st3in234',
1370                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1371                 'upload_date': '20130730',
1372                 'track': 'Such mich find mich',
1373                 'age_limit': 0,
1374                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1375                 'like_count': int,
1376                 'playable_in_embed': False,
1377                 'creator': 'OOMPH!',
1378                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1379                 'view_count': int,
1380                 'alt_title': 'Such mich find mich',
1381                 'duration': 210,
1382                 'channel': 'Herr Lurik',
1383                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1384                 'categories': ['Music'],
1385                 'availability': 'public',
1386                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1387                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1388                 'live_status': 'not_live',
1389                 'artist': 'OOMPH!',
1390                 'channel_follower_count': int
1391             },
1392         },
1393         {
1394             'note': 'Non-bypassable age-gated video',
1395             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1396             'only_matching': True,
1397         },
1398         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1399         # YouTube Red ad is not captured for creator
1400         {
1401             'url': '__2ABJjxzNo',
1402             'info_dict': {
1403                 'id': '__2ABJjxzNo',
1404                 'ext': 'mp4',
1405                 'duration': 266,
1406                 'upload_date': '20100430',
1407                 'uploader_id': 'deadmau5',
1408                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1409                 'creator': 'deadmau5',
1410                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1411                 'uploader': 'deadmau5',
1412                 'title': 'Deadmau5 - Some Chords (HD)',
1413                 'alt_title': 'Some Chords',
1414                 'availability': 'public',
1415                 'tags': 'count:14',
1416                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1417                 'view_count': int,
1418                 'live_status': 'not_live',
1419                 'channel': 'deadmau5',
1420                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1421                 'like_count': int,
1422                 'track': 'Some Chords',
1423                 'artist': 'deadmau5',
1424                 'playable_in_embed': True,
1425                 'age_limit': 0,
1426                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1427                 'categories': ['Music'],
1428                 'album': 'Some Chords',
1429                 'channel_follower_count': int
1430             },
1431             'expected_warnings': [
1432                 'DASH manifest missing',
1433             ]
1434         },
1435         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1436         {
1437             'url': 'lqQg6PlCWgI',
1438             'info_dict': {
1439                 'id': 'lqQg6PlCWgI',
1440                 'ext': 'mp4',
1441                 'duration': 6085,
1442                 'upload_date': '20150827',
1443                 'uploader_id': 'olympic',
1444                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1445                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1446                 'uploader': 'Olympics',
1447                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1448                 'like_count': int,
1449                 'release_timestamp': 1343767800,
1450                 'playable_in_embed': True,
1451                 'categories': ['Sports'],
1452                 'release_date': '20120731',
1453                 'channel': 'Olympics',
1454                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1455                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1456                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1457                 'age_limit': 0,
1458                 'availability': 'public',
1459                 'live_status': 'was_live',
1460                 'view_count': int,
1461                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1462                 'channel_follower_count': int
1463             },
1464             'params': {
1465                 'skip_download': 'requires avconv',
1466             }
1467         },
1468         # Non-square pixels
1469         {
1470             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1471             'info_dict': {
1472                 'id': '_b-2C3KPAM0',
1473                 'ext': 'mp4',
1474                 'stretched_ratio': 16 / 9.,
1475                 'duration': 85,
1476                 'upload_date': '20110310',
1477                 'uploader_id': 'AllenMeow',
1478                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1479                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1480                 'uploader': '孫ᄋᄅ',
1481                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1482                 'playable_in_embed': True,
1483                 'channel': '孫ᄋᄅ',
1484                 'age_limit': 0,
1485                 'tags': 'count:11',
1486                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1487                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1488                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1489                 'view_count': int,
1490                 'categories': ['People & Blogs'],
1491                 'like_count': int,
1492                 'live_status': 'not_live',
1493                 'availability': 'unlisted',
1494                 'comment_count': int,
1495                 'channel_follower_count': int
1496             },
1497         },
1498         # url_encoded_fmt_stream_map is empty string
1499         {
1500             'url': 'qEJwOuvDf7I',
1501             'info_dict': {
1502                 'id': 'qEJwOuvDf7I',
1503                 'ext': 'webm',
1504                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1505                 'description': '',
1506                 'upload_date': '20150404',
1507                 'uploader_id': 'spbelect',
1508                 'uploader': 'Наблюдатели Петербурга',
1509             },
1510             'params': {
1511                 'skip_download': 'requires avconv',
1512             },
1513             'skip': 'This live event has ended.',
1514         },
1515         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1516         {
1517             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1518             'info_dict': {
1519                 'id': 'FIl7x6_3R5Y',
1520                 'ext': 'webm',
1521                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1522                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1523                 'duration': 220,
1524                 'upload_date': '20150625',
1525                 'uploader_id': 'dorappi2000',
1526                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1527                 'uploader': 'dorappi2000',
1528                 'formats': 'mincount:31',
1529             },
1530             'skip': 'not actual anymore',
1531         },
1532         # DASH manifest with segment_list
1533         {
1534             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1535             'md5': '8ce563a1d667b599d21064e982ab9e31',
1536             'info_dict': {
1537                 'id': 'CsmdDsKjzN8',
1538                 'ext': 'mp4',
1539                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1540                 'uploader': 'Airtek',
1541                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1542                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1543                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1544             },
1545             'params': {
1546                 'youtube_include_dash_manifest': True,
1547                 'format': '135',  # bestvideo
1548             },
1549             'skip': 'This live event has ended.',
1550         },
1551         {
1552             # Multifeed videos (multiple cameras), URL is for Main Camera
1553             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1554             'info_dict': {
1555                 'id': 'jvGDaLqkpTg',
1556                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1557                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1558             },
1559             'playlist': [{
1560                 'info_dict': {
1561                     'id': 'jvGDaLqkpTg',
1562                     'ext': 'mp4',
1563                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1564                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1565                     'duration': 10643,
1566                     'upload_date': '20161111',
1567                     'uploader': 'Team PGP',
1568                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1569                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1570                 },
1571             }, {
1572                 'info_dict': {
1573                     'id': '3AKt1R1aDnw',
1574                     'ext': 'mp4',
1575                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1576                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1577                     'duration': 10991,
1578                     'upload_date': '20161111',
1579                     'uploader': 'Team PGP',
1580                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1581                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1582                 },
1583             }, {
1584                 'info_dict': {
1585                     'id': 'RtAMM00gpVc',
1586                     'ext': 'mp4',
1587                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1588                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1589                     'duration': 10995,
1590                     'upload_date': '20161111',
1591                     'uploader': 'Team PGP',
1592                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1593                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1594                 },
1595             }, {
1596                 'info_dict': {
1597                     'id': '6N2fdlP3C5U',
1598                     'ext': 'mp4',
1599                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1600                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1601                     'duration': 10990,
1602                     'upload_date': '20161111',
1603                     'uploader': 'Team PGP',
1604                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1605                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1606                 },
1607             }],
1608             'params': {
1609                 'skip_download': True,
1610             },
1611             'skip': 'Not multifeed anymore',
1612         },
1613         {
1614             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1615             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1616             'info_dict': {
1617                 'id': 'gVfLd0zydlo',
1618                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1619             },
1620             'playlist_count': 2,
1621             'skip': 'Not multifeed anymore',
1622         },
1623         {
1624             'url': 'https://vid.plus/FlRa-iH7PGw',
1625             'only_matching': True,
1626         },
1627         {
1628             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1629             'only_matching': True,
1630         },
1631         {
1632             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1633             # Also tests cut-off URL expansion in video description (see
1634             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1635             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1636             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1637             'info_dict': {
1638                 'id': 'lsguqyKfVQg',
1639                 'ext': 'mp4',
1640                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1641                 'alt_title': 'Dark Walk',
1642                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1643                 'duration': 133,
1644                 'upload_date': '20151119',
1645                 'uploader_id': 'IronSoulElf',
1646                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1647                 'uploader': 'IronSoulElf',
1648                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1649                 'track': 'Dark Walk',
1650                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1651                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1652                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1653                 'categories': ['Film & Animation'],
1654                 'view_count': int,
1655                 'live_status': 'not_live',
1656                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1657                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1658                 'tags': 'count:13',
1659                 'availability': 'public',
1660                 'channel': 'IronSoulElf',
1661                 'playable_in_embed': True,
1662                 'like_count': int,
1663                 'age_limit': 0,
1664                 'channel_follower_count': int
1665             },
1666             'params': {
1667                 'skip_download': True,
1668             },
1669         },
1670         {
1671             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1672             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1673             'only_matching': True,
1674         },
1675         {
1676             # Video with yt:stretch=17:0
1677             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1678             'info_dict': {
1679                 'id': 'Q39EVAstoRM',
1680                 'ext': 'mp4',
1681                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1682                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1683                 'upload_date': '20151107',
1684                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1685                 'uploader': 'CH GAMER DROID',
1686             },
1687             'params': {
1688                 'skip_download': True,
1689             },
1690             'skip': 'This video does not exist.',
1691         },
1692         {
1693             # Video with incomplete 'yt:stretch=16:'
1694             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1695             'only_matching': True,
1696         },
1697         {
1698             # Video licensed under Creative Commons
1699             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1700             'info_dict': {
1701                 'id': 'M4gD1WSo5mA',
1702                 'ext': 'mp4',
1703                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1704                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1705                 'duration': 721,
1706                 'upload_date': '20150128',
1707                 'uploader_id': 'BerkmanCenter',
1708                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1709                 'uploader': 'The Berkman Klein Center for Internet & Society',
1710                 'license': 'Creative Commons Attribution license (reuse allowed)',
1711                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1712                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1713                 'like_count': int,
1714                 'age_limit': 0,
1715                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1716                 'channel': 'The Berkman Klein Center for Internet & Society',
1717                 'availability': 'public',
1718                 'view_count': int,
1719                 'categories': ['Education'],
1720                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1721                 'live_status': 'not_live',
1722                 'playable_in_embed': True,
1723                 'comment_count': int,
1724                 'channel_follower_count': int
1725             },
1726             'params': {
1727                 'skip_download': True,
1728             },
1729         },
1730         {
1731             # Channel-like uploader_url
1732             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1733             'info_dict': {
1734                 'id': 'eQcmzGIKrzg',
1735                 'ext': 'mp4',
1736                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1737                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1738                 'duration': 4060,
1739                 'upload_date': '20151120',
1740                 'uploader': 'Bernie Sanders',
1741                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1742                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1743                 'license': 'Creative Commons Attribution license (reuse allowed)',
1744                 'playable_in_embed': True,
1745                 'tags': 'count:12',
1746                 'like_count': int,
1747                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1748                 'age_limit': 0,
1749                 'availability': 'public',
1750                 'categories': ['News & Politics'],
1751                 'channel': 'Bernie Sanders',
1752                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1753                 'view_count': int,
1754                 'live_status': 'not_live',
1755                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1756                 'comment_count': int,
1757                 'channel_follower_count': int
1758             },
1759             'params': {
1760                 'skip_download': True,
1761             },
1762         },
1763         {
1764             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1765             'only_matching': True,
1766         },
1767         {
1768             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1769             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1770             'only_matching': True,
1771         },
1772         {
1773             # Rental video preview
1774             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1775             'info_dict': {
1776                 'id': 'uGpuVWrhIzE',
1777                 'ext': 'mp4',
1778                 'title': 'Piku - Trailer',
1779                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1780                 'upload_date': '20150811',
1781                 'uploader': 'FlixMatrix',
1782                 'uploader_id': 'FlixMatrixKaravan',
1783                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1784                 'license': 'Standard YouTube License',
1785             },
1786             'params': {
1787                 'skip_download': True,
1788             },
1789             'skip': 'This video is not available.',
1790         },
1791         {
1792             # YouTube Red video with episode data
1793             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1794             'info_dict': {
1795                 'id': 'iqKdEhx-dD4',
1796                 'ext': 'mp4',
1797                 'title': 'Isolation - Mind Field (Ep 1)',
1798                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1799                 'duration': 2085,
1800                 'upload_date': '20170118',
1801                 'uploader': 'Vsauce',
1802                 'uploader_id': 'Vsauce',
1803                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1804                 'series': 'Mind Field',
1805                 'season_number': 1,
1806                 'episode_number': 1,
1807                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1808                 'tags': 'count:12',
1809                 'view_count': int,
1810                 'availability': 'public',
1811                 'age_limit': 0,
1812                 'channel': 'Vsauce',
1813                 'episode': 'Episode 1',
1814                 'categories': ['Entertainment'],
1815                 'season': 'Season 1',
1816                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1817                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1818                 'like_count': int,
1819                 'playable_in_embed': True,
1820                 'live_status': 'not_live',
1821                 'channel_follower_count': int
1822             },
1823             'params': {
1824                 'skip_download': True,
1825             },
1826             'expected_warnings': [
1827                 'Skipping DASH manifest',
1828             ],
1829         },
1830         {
1831             # The following content has been identified by the YouTube community
1832             # as inappropriate or offensive to some audiences.
1833             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1834             'info_dict': {
1835                 'id': '6SJNVb0GnPI',
1836                 'ext': 'mp4',
1837                 'title': 'Race Differences in Intelligence',
1838                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1839                 'duration': 965,
1840                 'upload_date': '20140124',
1841                 'uploader': 'New Century Foundation',
1842                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1843                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1844             },
1845             'params': {
1846                 'skip_download': True,
1847             },
1848             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1849         },
1850         {
1851             # itag 212
1852             'url': '1t24XAntNCY',
1853             'only_matching': True,
1854         },
1855         {
1856             # geo restricted to JP
1857             'url': 'sJL6WA-aGkQ',
1858             'only_matching': True,
1859         },
1860         {
1861             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1862             'only_matching': True,
1863         },
1864         {
1865             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1866             'only_matching': True,
1867         },
1868         {
1869             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1870             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1871             'only_matching': True,
1872         },
1873         {
1874             # DRM protected
1875             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1876             'only_matching': True,
1877         },
1878         {
1879             # Video with unsupported adaptive stream type formats
1880             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1881             'info_dict': {
1882                 'id': 'Z4Vy8R84T1U',
1883                 'ext': 'mp4',
1884                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1885                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1886                 'duration': 433,
1887                 'upload_date': '20130923',
1888                 'uploader': 'Amelia Putri Harwita',
1889                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1890                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1891                 'formats': 'maxcount:10',
1892             },
1893             'params': {
1894                 'skip_download': True,
1895                 'youtube_include_dash_manifest': False,
1896             },
1897             'skip': 'not actual anymore',
1898         },
1899         {
1900             # Youtube Music Auto-generated description
1901             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1902             'info_dict': {
1903                 'id': 'MgNrAu2pzNs',
1904                 'ext': 'mp4',
1905                 'title': 'Voyeur Girl',
1906                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1907                 'upload_date': '20190312',
1908                 'uploader': 'Stephen - Topic',
1909                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1910                 'artist': 'Stephen',
1911                 'track': 'Voyeur Girl',
1912                 'album': 'it\'s too much love to know my dear',
1913                 'release_date': '20190313',
1914                 'release_year': 2019,
1915                 'alt_title': 'Voyeur Girl',
1916                 'view_count': int,
1917                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1918                 'playable_in_embed': True,
1919                 'like_count': int,
1920                 'categories': ['Music'],
1921                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1922                 'channel': 'Stephen',
1923                 'availability': 'public',
1924                 'creator': 'Stephen',
1925                 'duration': 169,
1926                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1927                 'age_limit': 0,
1928                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1929                 'tags': 'count:11',
1930                 'live_status': 'not_live',
1931                 'channel_follower_count': int
1932             },
1933             'params': {
1934                 'skip_download': True,
1935             },
1936         },
1937         {
1938             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1939             'only_matching': True,
1940         },
1941         {
1942             # invalid -> valid video id redirection
1943             'url': 'DJztXj2GPfl',
1944             'info_dict': {
1945                 'id': 'DJztXj2GPfk',
1946                 'ext': 'mp4',
1947                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1948                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1949                 'upload_date': '20090125',
1950                 'uploader': 'Prochorowka',
1951                 'uploader_id': 'Prochorowka',
1952                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1953                 'artist': 'Panjabi MC',
1954                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1955                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1956             },
1957             'params': {
1958                 'skip_download': True,
1959             },
1960             'skip': 'Video unavailable',
1961         },
1962         {
1963             # empty description results in an empty string
1964             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1965             'info_dict': {
1966                 'id': 'x41yOUIvK2k',
1967                 'ext': 'mp4',
1968                 'title': 'IMG 3456',
1969                 'description': '',
1970                 'upload_date': '20170613',
1971                 'uploader_id': 'ElevageOrVert',
1972                 'uploader': 'ElevageOrVert',
1973                 'view_count': int,
1974                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1975                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1976                 'like_count': int,
1977                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1978                 'tags': [],
1979                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1980                 'availability': 'public',
1981                 'age_limit': 0,
1982                 'categories': ['Pets & Animals'],
1983                 'duration': 7,
1984                 'playable_in_embed': True,
1985                 'live_status': 'not_live',
1986                 'channel': 'ElevageOrVert',
1987                 'channel_follower_count': int
1988             },
1989             'params': {
1990                 'skip_download': True,
1991             },
1992         },
1993         {
1994             # with '};' inside yt initial data (see [1])
1995             # see [2] for an example with '};' inside ytInitialPlayerResponse
1996             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1997             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1998             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1999             'info_dict': {
2000                 'id': 'CHqg6qOn4no',
2001                 'ext': 'mp4',
2002                 'title': 'Part 77   Sort a list of simple types in c#',
2003                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2004                 'upload_date': '20130831',
2005                 'uploader_id': 'kudvenkat',
2006                 'uploader': 'kudvenkat',
2007                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2008                 'like_count': int,
2009                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2010                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2011                 'live_status': 'not_live',
2012                 'categories': ['Education'],
2013                 'availability': 'public',
2014                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2015                 'tags': 'count:12',
2016                 'playable_in_embed': True,
2017                 'age_limit': 0,
2018                 'view_count': int,
2019                 'duration': 522,
2020                 'channel': 'kudvenkat',
2021                 'comment_count': int,
2022                 'channel_follower_count': int
2023             },
2024             'params': {
2025                 'skip_download': True,
2026             },
2027         },
2028         {
2029             # another example of '};' in ytInitialData
2030             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2031             'only_matching': True,
2032         },
2033         {
2034             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2035             'only_matching': True,
2036         },
2037         {
2038             # https://github.com/ytdl-org/youtube-dl/pull/28094
2039             'url': 'OtqTfy26tG0',
2040             'info_dict': {
2041                 'id': 'OtqTfy26tG0',
2042                 'ext': 'mp4',
2043                 'title': 'Burn Out',
2044                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2045                 'upload_date': '20141120',
2046                 'uploader': 'The Cinematic Orchestra - Topic',
2047                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2048                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2049                 'artist': 'The Cinematic Orchestra',
2050                 'track': 'Burn Out',
2051                 'album': 'Every Day',
2052                 'like_count': int,
2053                 'live_status': 'not_live',
2054                 'alt_title': 'Burn Out',
2055                 'duration': 614,
2056                 'age_limit': 0,
2057                 'view_count': int,
2058                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2059                 'creator': 'The Cinematic Orchestra',
2060                 'channel': 'The Cinematic Orchestra',
2061                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2062                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2063                 'availability': 'public',
2064                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2065                 'categories': ['Music'],
2066                 'playable_in_embed': True,
2067                 'channel_follower_count': int
2068             },
2069             'params': {
2070                 'skip_download': True,
2071             },
2072         },
2073         {
2074             # controversial video, only works with bpctr when authenticated with cookies
2075             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2076             'only_matching': True,
2077         },
2078         {
2079             # controversial video, requires bpctr/contentCheckOk
2080             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2081             'info_dict': {
2082                 'id': 'SZJvDhaSDnc',
2083                 'ext': 'mp4',
2084                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2085                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2086                 'uploader': 'CBS Mornings',
2087                 'uploader_id': 'CBSThisMorning',
2088                 'upload_date': '20140716',
2089                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2090                 'duration': 170,
2091                 'categories': ['News & Politics'],
2092                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2093                 'view_count': int,
2094                 'channel': 'CBS Mornings',
2095                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2096                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2097                 'age_limit': 18,
2098                 'availability': 'needs_auth',
2099                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2100                 'like_count': int,
2101                 'live_status': 'not_live',
2102                 'playable_in_embed': True,
2103                 'channel_follower_count': int
2104             }
2105         },
2106         {
2107             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2108             'url': 'cBvYw8_A0vQ',
2109             'info_dict': {
2110                 'id': 'cBvYw8_A0vQ',
2111                 'ext': 'mp4',
2112                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2113                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2114                 'upload_date': '20201120',
2115                 'uploader': 'Walk around Japan',
2116                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2117                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2118                 'duration': 1456,
2119                 'categories': ['Travel & Events'],
2120                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2121                 'view_count': int,
2122                 'channel': 'Walk around Japan',
2123                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2124                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2125                 'age_limit': 0,
2126                 'availability': 'public',
2127                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2128                 'live_status': 'not_live',
2129                 'playable_in_embed': True,
2130                 'channel_follower_count': int
2131             },
2132             'params': {
2133                 'skip_download': True,
2134             },
2135         }, {
2136             # Has multiple audio streams
2137             'url': 'WaOKSUlf4TM',
2138             'only_matching': True
2139         }, {
2140             # Requires Premium: has format 141 when requested using YTM url
2141             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2142             'only_matching': True
2143         }, {
2144             # multiple subtitles with same lang_code
2145             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2146             'only_matching': True,
2147         }, {
2148             # Force use android client fallback
2149             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2150             'info_dict': {
2151                 'id': 'YOelRv7fMxY',
2152                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2153                 'ext': '3gp',
2154                 'upload_date': '20210624',
2155                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2156                 'uploader': 'colinfurze',
2157                 'uploader_id': 'colinfurze',
2158                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2159                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2160                 'duration': 596,
2161                 'categories': ['Entertainment'],
2162                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2163                 'view_count': int,
2164                 'channel': 'colinfurze',
2165                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2166                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2167                 'age_limit': 0,
2168                 'availability': 'public',
2169                 'like_count': int,
2170                 'live_status': 'not_live',
2171                 'playable_in_embed': True,
2172                 'channel_follower_count': int
2173             },
2174             'params': {
2175                 'format': '17',  # 3gp format available on android
2176                 'extractor_args': {'youtube': {'player_client': ['android']}},
2177             },
2178         },
2179         {
2180             # Skip download of additional client configs (remix client config in this case)
2181             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2182             'only_matching': True,
2183             'params': {
2184                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2185             },
2186         }, {
2187             # shorts
2188             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2189             'only_matching': True,
2190         }, {
2191             'note': 'Storyboards',
2192             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2193             'info_dict': {
2194                 'id': '5KLPxDtMqe8',
2195                 'ext': 'mhtml',
2196                 'format_id': 'sb0',
2197                 'title': 'Your Brain is Plastic',
2198                 'uploader_id': 'scishow',
2199                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2200                 'upload_date': '20140324',
2201                 'uploader': 'SciShow',
2202                 'like_count': int,
2203                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2204                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2205                 'view_count': int,
2206                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2207                 'playable_in_embed': True,
2208                 'tags': 'count:12',
2209                 'uploader_url': 'http://www.youtube.com/user/scishow',
2210                 'availability': 'public',
2211                 'channel': 'SciShow',
2212                 'live_status': 'not_live',
2213                 'duration': 248,
2214                 'categories': ['Education'],
2215                 'age_limit': 0,
2216                 'channel_follower_count': int
2217             }, 'params': {'format': 'mhtml', 'skip_download': True}
2218         }, {
2219             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2220             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2221             'info_dict': {
2222                 'id': '2NUZ8W2llS4',
2223                 'ext': 'mp4',
2224                 'title': 'The NP that test your phone performance 🙂',
2225                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2226                 'uploader': 'Leon Nguyen',
2227                 'uploader_id': 'VNSXIII',
2228                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2229                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2230                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2231                 'duration': 21,
2232                 'view_count': int,
2233                 'age_limit': 0,
2234                 'categories': ['Gaming'],
2235                 'tags': 'count:23',
2236                 'playable_in_embed': True,
2237                 'live_status': 'not_live',
2238                 'upload_date': '20220103',
2239                 'like_count': int,
2240                 'availability': 'public',
2241                 'channel': 'Leon Nguyen',
2242                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2243                 'comment_count': int,
2244                 'channel_follower_count': int
2245             }
2246         }, {
2247             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2248             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2249             'info_dict': {
2250                 'id': '2NUZ8W2llS4',
2251                 'ext': 'mp4',
2252                 'title': 'The NP that test your phone performance 🙂',
2253                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2254                 'uploader': 'Leon Nguyen',
2255                 'uploader_id': 'VNSXIII',
2256                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2257                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2258                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2259                 'duration': 21,
2260                 'view_count': int,
2261                 'age_limit': 0,
2262                 'categories': ['Gaming'],
2263                 'tags': 'count:23',
2264                 'playable_in_embed': True,
2265                 'live_status': 'not_live',
2266                 'upload_date': '20220102',
2267                 'like_count': int,
2268                 'availability': 'public',
2269                 'channel': 'Leon Nguyen',
2270                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2271                 'comment_count': int,
2272                 'channel_follower_count': int
2273             },
2274             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2275         }, {
2276             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2277             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2278             'info_dict': {
2279                 'id': 'mzZzzBU6lrM',
2280                 'ext': 'mp4',
2281                 'title': 'I Met GeorgeNotFound In Real Life...',
2282                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2283                 'uploader': 'Quackity',
2284                 'uploader_id': 'QuackityHQ',
2285                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2286                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2287                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2288                 'duration': 955,
2289                 'view_count': int,
2290                 'age_limit': 0,
2291                 'categories': ['Entertainment'],
2292                 'tags': 'count:26',
2293                 'playable_in_embed': True,
2294                 'live_status': 'not_live',
2295                 'release_timestamp': 1641172509,
2296                 'release_date': '20220103',
2297                 'upload_date': '20220103',
2298                 'like_count': int,
2299                 'availability': 'public',
2300                 'channel': 'Quackity',
2301                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2302                 'channel_follower_count': int
2303             }
2304         },
2305         {   # continuous livestream. Microformat upload date should be preferred.
2306             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2307             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2308             'info_dict': {
2309                 'id': 'kgx4WGK0oNU',
2310                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2311                 'ext': 'mp4',
2312                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2313                 'availability': 'public',
2314                 'age_limit': 0,
2315                 'release_timestamp': 1637975704,
2316                 'upload_date': '20210619',
2317                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2318                 'live_status': 'is_live',
2319                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2320                 'uploader': '阿鲍Abao',
2321                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2322                 'channel': 'Abao in Tokyo',
2323                 'channel_follower_count': int,
2324                 'release_date': '20211127',
2325                 'tags': 'count:39',
2326                 'categories': ['People & Blogs'],
2327                 'like_count': int,
2328                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2329                 'view_count': int,
2330                 'playable_in_embed': True,
2331                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2332                 'concurrent_view_count': int,
2333             },
2334             'params': {'skip_download': True}
2335         }, {
2336             # Story. Requires specific player params to work.
2337             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2338             'info_dict': {
2339                 'id': 'vv8qTUWmulI',
2340                 'ext': 'mp4',
2341                 'availability': 'unlisted',
2342                 'view_count': int,
2343                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2344                 'upload_date': '20220526',
2345                 'categories': ['Education'],
2346                 'title': 'Story',
2347                 'channel': 'IT\'S HISTORY',
2348                 'description': '',
2349                 'uploader_id': 'BlastfromthePast',
2350                 'duration': 12,
2351                 'uploader': 'IT\'S HISTORY',
2352                 'playable_in_embed': True,
2353                 'age_limit': 0,
2354                 'live_status': 'not_live',
2355                 'tags': [],
2356                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2357                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2358                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2359             },
2360             'skip': 'stories get removed after some period of time',
2361         }, {
2362             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2363             'info_dict': {
2364                 'id': 'tjjjtzRLHvA',
2365                 'ext': 'mp4',
2366                 'title': 'ハッシュタグ無し };if window.ytcsi',
2367                 'upload_date': '20220323',
2368                 'like_count': int,
2369                 'availability': 'unlisted',
2370                 'channel': 'nao20010128nao',
2371                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2372                 'age_limit': 0,
2373                 'uploader': 'nao20010128nao',
2374                 'uploader_id': 'nao20010128nao',
2375                 'categories': ['Music'],
2376                 'view_count': int,
2377                 'description': '',
2378                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2379                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2380                 'live_status': 'not_live',
2381                 'playable_in_embed': True,
2382                 'channel_follower_count': int,
2383                 'duration': 6,
2384                 'tags': [],
2385                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2386             }
2387         }, {
2388             # Prefer primary title+description language metadata by default
2389             # Do not prefer translated description if primary is empty
2390             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2391             'info_dict': {
2392                 'id': 'el3E4MbxRqQ',
2393                 'ext': 'mp4',
2394                 'title': 'dlp test video 2 - primary sv no desc',
2395                 'description': '',
2396                 'channel': 'cole-dlp-test-acc',
2397                 'tags': [],
2398                 'view_count': int,
2399                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2400                 'like_count': int,
2401                 'playable_in_embed': True,
2402                 'availability': 'unlisted',
2403                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2404                 'age_limit': 0,
2405                 'duration': 5,
2406                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2407                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2408                 'live_status': 'not_live',
2409                 'upload_date': '20220908',
2410                 'categories': ['People & Blogs'],
2411                 'uploader': 'cole-dlp-test-acc',
2412                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2413             },
2414             'params': {'skip_download': True}
2415         }, {
2416             # Extractor argument: prefer translated title+description
2417             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2418             'info_dict': {
2419                 'id': 'gHKT4uU8Zng',
2420                 'ext': 'mp4',
2421                 'channel': 'cole-dlp-test-acc',
2422                 'tags': [],
2423                 'duration': 5,
2424                 'live_status': 'not_live',
2425                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2426                 'upload_date': '20220728',
2427                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2428                 'view_count': int,
2429                 'categories': ['People & Blogs'],
2430                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2431                 'title': 'dlp test video title translated (fr)',
2432                 'availability': 'public',
2433                 'uploader': 'cole-dlp-test-acc',
2434                 'age_limit': 0,
2435                 'description': 'dlp test video description translated (fr)',
2436                 'playable_in_embed': True,
2437                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2438                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2439             },
2440             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2441             'expected_warnings': [r'Preferring "fr" translated fields'],
2442         }, {
2443             'note': '6 channel audio',
2444             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2445             'only_matching': True,
2446         }
2447     ]
2448
2449     _WEBPAGE_TESTS = [
2450         # YouTube <object> embed
2451         {
2452             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2453             'md5': '873c81d308b979f0e23ee7e620b312a3',
2454             'info_dict': {
2455                 'id': 'msN87y-iEx0',
2456                 'ext': 'mp4',
2457                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2458                 'upload_date': '20080526',
2459                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2460                 'uploader': 'Christopher Sykes',
2461                 'uploader_id': 'ChristopherJSykes',
2462                 'age_limit': 0,
2463                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2464                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2465                 'playable_in_embed': True,
2466                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2467                 'like_count': int,
2468                 'comment_count': int,
2469                 'channel': 'Christopher Sykes',
2470                 'live_status': 'not_live',
2471                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2472                 'availability': 'public',
2473                 'duration': 195,
2474                 'view_count': int,
2475                 'categories': ['Science & Technology'],
2476                 'channel_follower_count': int,
2477                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2478             },
2479             'params': {
2480                 'skip_download': True,
2481             }
2482         },
2483     ]
2484
2485     @classmethod
2486     def suitable(cls, url):
2487         from ..utils import parse_qs
2488
2489         qs = parse_qs(url)
2490         if qs.get('list', [None])[0]:
2491             return False
2492         return super().suitable(url)
2493
2494     def __init__(self, *args, **kwargs):
2495         super().__init__(*args, **kwargs)
2496         self._code_cache = {}
2497         self._player_cache = {}
2498
2499     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2500         lock = threading.Lock()
2501         start_time = time.time()
2502         formats = [f for f in formats if f.get('is_from_start')]
2503
2504         def refetch_manifest(format_id, delay):
2505             nonlocal formats, start_time, is_live
2506             if time.time() <= start_time + delay:
2507                 return
2508
2509             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2510             video_details = traverse_obj(
2511                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2512             microformats = traverse_obj(
2513                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2514                 expected_type=dict, default=[])
2515             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2516             is_live = live_status == 'is_live'
2517             start_time = time.time()
2518
2519         def mpd_feed(format_id, delay):
2520             """
2521             @returns (manifest_url, manifest_stream_number, is_live) or None
2522             """
2523             with lock:
2524                 refetch_manifest(format_id, delay)
2525
2526             f = next((f for f in formats if f['format_id'] == format_id), None)
2527             if not f:
2528                 if not is_live:
2529                     self.to_screen(f'{video_id}: Video is no longer live')
2530                 else:
2531                     self.report_warning(
2532                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2533                 return None
2534             return f['manifest_url'], f['manifest_stream_number'], is_live
2535
2536         for f in formats:
2537             f['is_live'] = is_live
2538             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2539                                     live_start_time, mpd_feed, not is_live and f.copy())
2540             if is_live:
2541                 f['fragments'] = gen
2542                 f['protocol'] = 'http_dash_segments_generator'
2543             else:
2544                 f['fragments'] = LazyList(gen({}))
2545                 del f['is_from_start']
2546
2547     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2548         FETCH_SPAN, MAX_DURATION = 5, 432000
2549
2550         mpd_url, stream_number, is_live = None, None, True
2551
2552         begin_index = 0
2553         download_start_time = ctx.get('start') or time.time()
2554
2555         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2556         if lack_early_segments:
2557             self.report_warning(bug_reports_message(
2558                 'Starting download from the last 120 hours of the live stream since '
2559                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2560             lack_early_segments = True
2561
2562         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2563         fragments, fragment_base_url = None, None
2564
2565         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2566             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2567             # Obtain from MPD's maximum seq value
2568             old_mpd_url = mpd_url
2569             last_error = ctx.pop('last_error', None)
2570             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2571             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2572                                                or (mpd_url, stream_number, False))
2573             if not refresh_sequence:
2574                 if expire_fast and not is_live:
2575                     return False, last_seq
2576                 elif old_mpd_url == mpd_url:
2577                     return True, last_seq
2578             if manifestless_orig_fmt:
2579                 fmt_info = manifestless_orig_fmt
2580             else:
2581                 try:
2582                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2583                         mpd_url, None, note=False, errnote=False, fatal=False)
2584                 except ExtractorError:
2585                     fmts = None
2586                 if not fmts:
2587                     no_fragment_score += 2
2588                     return False, last_seq
2589                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2590             fragments = fmt_info['fragments']
2591             fragment_base_url = fmt_info['fragment_base_url']
2592             assert fragment_base_url
2593
2594             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2595             return True, _last_seq
2596
2597         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2598         while is_live:
2599             fetch_time = time.time()
2600             if no_fragment_score > 30:
2601                 return
2602             if last_segment_url:
2603                 # Obtain from "X-Head-Seqnum" header value from each segment
2604                 try:
2605                     urlh = self._request_webpage(
2606                         last_segment_url, None, note=False, errnote=False, fatal=False)
2607                 except ExtractorError:
2608                     urlh = None
2609                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2610                 if last_seq is None:
2611                     no_fragment_score += 2
2612                     last_segment_url = None
2613                     continue
2614             else:
2615                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2616                 no_fragment_score += 2
2617                 if not should_continue:
2618                     continue
2619
2620             if known_idx > last_seq:
2621                 last_segment_url = None
2622                 continue
2623
2624             last_seq += 1
2625
2626             if begin_index < 0 and known_idx < 0:
2627                 # skip from the start when it's negative value
2628                 known_idx = last_seq + begin_index
2629             if lack_early_segments:
2630                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2631             try:
2632                 for idx in range(known_idx, last_seq):
2633                     # do not update sequence here or you'll get skipped some part of it
2634                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2635                     if not should_continue:
2636                         known_idx = idx - 1
2637                         raise ExtractorError('breaking out of outer loop')
2638                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2639                     yield {
2640                         'url': last_segment_url,
2641                         'fragment_count': last_seq,
2642                     }
2643                 if known_idx == last_seq:
2644                     no_fragment_score += 5
2645                 else:
2646                     no_fragment_score = 0
2647                 known_idx = last_seq
2648             except ExtractorError:
2649                 continue
2650
2651             if manifestless_orig_fmt:
2652                 # Stop at the first iteration if running for post-live manifestless;
2653                 # fragment count no longer increase since it starts
2654                 break
2655
2656             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2657
2658     def _extract_player_url(self, *ytcfgs, webpage=None):
2659         player_url = traverse_obj(
2660             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2661             get_all=False, expected_type=str)
2662         if not player_url:
2663             return
2664         return urljoin('https://www.youtube.com', player_url)
2665
2666     def _download_player_url(self, video_id, fatal=False):
2667         res = self._download_webpage(
2668             'https://www.youtube.com/iframe_api',
2669             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2670         if res:
2671             player_version = self._search_regex(
2672                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2673             if player_version:
2674                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2675
2676     def _signature_cache_id(self, example_sig):
2677         """ Return a string representation of a signature """
2678         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2679
2680     @classmethod
2681     def _extract_player_info(cls, player_url):
2682         for player_re in cls._PLAYER_INFO_RE:
2683             id_m = re.search(player_re, player_url)
2684             if id_m:
2685                 break
2686         else:
2687             raise ExtractorError('Cannot identify player %r' % player_url)
2688         return id_m.group('id')
2689
2690     def _load_player(self, video_id, player_url, fatal=True):
2691         player_id = self._extract_player_info(player_url)
2692         if player_id not in self._code_cache:
2693             code = self._download_webpage(
2694                 player_url, video_id, fatal=fatal,
2695                 note='Downloading player ' + player_id,
2696                 errnote='Download of %s failed' % player_url)
2697             if code:
2698                 self._code_cache[player_id] = code
2699         return self._code_cache.get(player_id)
2700
2701     def _extract_signature_function(self, video_id, player_url, example_sig):
2702         player_id = self._extract_player_info(player_url)
2703
2704         # Read from filesystem cache
2705         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2706         assert os.path.basename(func_id) == func_id
2707
2708         self.write_debug(f'Extracting signature function {func_id}')
2709         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2710
2711         if not cache_spec:
2712             code = self._load_player(video_id, player_url)
2713         if code:
2714             res = self._parse_sig_js(code)
2715             test_string = ''.join(map(chr, range(len(example_sig))))
2716             cache_spec = [ord(c) for c in res(test_string)]
2717             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2718
2719         return lambda s: ''.join(s[i] for i in cache_spec)
2720
2721     def _print_sig_code(self, func, example_sig):
2722         if not self.get_param('youtube_print_sig_code'):
2723             return
2724
2725         def gen_sig_code(idxs):
2726             def _genslice(start, end, step):
2727                 starts = '' if start == 0 else str(start)
2728                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2729                 steps = '' if step == 1 else (':%d' % step)
2730                 return f's[{starts}{ends}{steps}]'
2731
2732             step = None
2733             # Quelch pyflakes warnings - start will be set when step is set
2734             start = '(Never used)'
2735             for i, prev in zip(idxs[1:], idxs[:-1]):
2736                 if step is not None:
2737                     if i - prev == step:
2738                         continue
2739                     yield _genslice(start, prev, step)
2740                     step = None
2741                     continue
2742                 if i - prev in [-1, 1]:
2743                     step = i - prev
2744                     start = prev
2745                     continue
2746                 else:
2747                     yield 's[%d]' % prev
2748             if step is None:
2749                 yield 's[%d]' % i
2750             else:
2751                 yield _genslice(start, i, step)
2752
2753         test_string = ''.join(map(chr, range(len(example_sig))))
2754         cache_res = func(test_string)
2755         cache_spec = [ord(c) for c in cache_res]
2756         expr_code = ' + '.join(gen_sig_code(cache_spec))
2757         signature_id_tuple = '(%s)' % (
2758             ', '.join(str(len(p)) for p in example_sig.split('.')))
2759         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2760                 '    return %s\n') % (signature_id_tuple, expr_code)
2761         self.to_screen('Extracted signature function:\n' + code)
2762
2763     def _parse_sig_js(self, jscode):
2764         funcname = self._search_regex(
2765             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2766              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2767              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2768              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2769              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2770              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2771              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2772              # Obsolete patterns
2773              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2774              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2775              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2776              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2777              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2778              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2779              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2780              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2781             jscode, 'Initial JS player signature function name', group='sig')
2782
2783         jsi = JSInterpreter(jscode)
2784         initial_function = jsi.extract_function(funcname)
2785         return lambda s: initial_function([s])
2786
2787     def _cached(self, func, *cache_id):
2788         def inner(*args, **kwargs):
2789             if cache_id not in self._player_cache:
2790                 try:
2791                     self._player_cache[cache_id] = func(*args, **kwargs)
2792                 except ExtractorError as e:
2793                     self._player_cache[cache_id] = e
2794                 except Exception as e:
2795                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
2796
2797             ret = self._player_cache[cache_id]
2798             if isinstance(ret, Exception):
2799                 raise ret
2800             return ret
2801         return inner
2802
2803     def _decrypt_signature(self, s, video_id, player_url):
2804         """Turn the encrypted s field into a working signature"""
2805         extract_sig = self._cached(
2806             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
2807         func = extract_sig(video_id, player_url, s)
2808         self._print_sig_code(func, s)
2809         return func(s)
2810
2811     def _decrypt_nsig(self, s, video_id, player_url):
2812         """Turn the encrypted n field into a working signature"""
2813         if player_url is None:
2814             raise ExtractorError('Cannot decrypt nsig without player_url')
2815         player_url = urljoin('https://www.youtube.com', player_url)
2816
2817         try:
2818             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
2819         except ExtractorError as e:
2820             raise ExtractorError('Unable to extract nsig function code', cause=e)
2821         if self.get_param('youtube_print_sig_code'):
2822             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2823
2824         try:
2825             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
2826             ret = extract_nsig(jsi, func_code)(s)
2827         except JSInterpreter.Exception as e:
2828             try:
2829                 jsi = PhantomJSwrapper(self, timeout=5000)
2830             except ExtractorError:
2831                 raise e
2832             self.report_warning(
2833                 f'Native nsig extraction failed: Trying with PhantomJS\n'
2834                 f'         n = {s} ; player = {player_url}', video_id)
2835             self.write_debug(e, only_once=True)
2836
2837             args, func_body = func_code
2838             ret = jsi.execute(
2839                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
2840                 video_id=video_id, note='Executing signature code').strip()
2841
2842         self.write_debug(f'Decrypted nsig {s} => {ret}')
2843         return ret
2844
2845     def _extract_n_function_name(self, jscode):
2846         funcname, idx = self._search_regex(
2847             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2848             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2849         if not idx:
2850             return funcname
2851
2852         return json.loads(js_to_json(self._search_regex(
2853             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
2854             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
2855
2856     def _extract_n_function_code(self, video_id, player_url):
2857         player_id = self._extract_player_info(player_url)
2858         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
2859         jscode = func_code or self._load_player(video_id, player_url)
2860         jsi = JSInterpreter(jscode)
2861
2862         if func_code:
2863             return jsi, player_id, func_code
2864
2865         func_name = self._extract_n_function_name(jscode)
2866
2867         # For redundancy
2868         func_code = self._search_regex(
2869             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
2870                      # NB: The end of the regex is intentionally kept strict
2871                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
2872             jscode, 'nsig function', group=('var', 'code'), default=None)
2873         if func_code:
2874             func_code = ([func_code[0]], func_code[1])
2875         else:
2876             self.write_debug('Extracting nsig function with jsinterp')
2877             func_code = jsi.extract_function_code(func_name)
2878
2879         self.cache.store('youtube-nsig', player_id, func_code)
2880         return jsi, player_id, func_code
2881
2882     def _extract_n_function_from_code(self, jsi, func_code):
2883         func = jsi.extract_function_from_code(*func_code)
2884
2885         def extract_nsig(s):
2886             try:
2887                 ret = func([s])
2888             except JSInterpreter.Exception:
2889                 raise
2890             except Exception as e:
2891                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
2892
2893             if ret.startswith('enhanced_except_'):
2894                 raise JSInterpreter.Exception('Signature function returned an exception')
2895             return ret
2896
2897         return extract_nsig
2898
2899     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2900         """
2901         Extract signatureTimestamp (sts)
2902         Required to tell API what sig/player version is in use.
2903         """
2904         sts = None
2905         if isinstance(ytcfg, dict):
2906             sts = int_or_none(ytcfg.get('STS'))
2907
2908         if not sts:
2909             # Attempt to extract from player
2910             if player_url is None:
2911                 error_msg = 'Cannot extract signature timestamp without player_url.'
2912                 if fatal:
2913                     raise ExtractorError(error_msg)
2914                 self.report_warning(error_msg)
2915                 return
2916             code = self._load_player(video_id, player_url, fatal=fatal)
2917             if code:
2918                 sts = int_or_none(self._search_regex(
2919                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2920                     'JS player signature timestamp', group='sts', fatal=fatal))
2921         return sts
2922
2923     def _mark_watched(self, video_id, player_responses):
2924         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2925             label = 'fully ' if is_full else ''
2926             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2927                             expected_type=url_or_none)
2928             if not url:
2929                 self.report_warning(f'Unable to mark {label}watched')
2930                 return
2931             parsed_url = urllib.parse.urlparse(url)
2932             qs = urllib.parse.parse_qs(parsed_url.query)
2933
2934             # cpn generation algorithm is reverse engineered from base.js.
2935             # In fact it works even with dummy cpn.
2936             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2937             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2938
2939             # # more consistent results setting it to right before the end
2940             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2941
2942             qs.update({
2943                 'ver': ['2'],
2944                 'cpn': [cpn],
2945                 'cmt': video_length,
2946                 'el': 'detailpage',  # otherwise defaults to "shorts"
2947             })
2948
2949             if is_full:
2950                 # these seem to mark watchtime "history" in the real world
2951                 # they're required, so send in a single value
2952                 qs.update({
2953                     'st': video_length,
2954                     'et': video_length,
2955                 })
2956
2957             url = urllib.parse.urlunparse(
2958                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2959
2960             self._download_webpage(
2961                 url, video_id, f'Marking {label}watched',
2962                 'Unable to mark watched', fatal=False)
2963
2964     @classmethod
2965     def _extract_from_webpage(cls, url, webpage):
2966         # Invidious Instances
2967         # https://github.com/yt-dlp/yt-dlp/issues/195
2968         # https://github.com/iv-org/invidious/pull/1730
2969         mobj = re.search(
2970             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
2971             webpage)
2972         if mobj:
2973             yield cls.url_result(mobj.group('url'), cls)
2974             raise cls.StopExtraction()
2975
2976         yield from super()._extract_from_webpage(url, webpage)
2977
2978         # lazyYT YouTube embed
2979         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
2980             yield cls.url_result(unescapeHTML(id_), cls, id_)
2981
2982         # Wordpress "YouTube Video Importer" plugin
2983         for m in re.findall(r'''(?x)<div[^>]+
2984                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2985                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
2986             yield cls.url_result(m[-1], cls, m[-1])
2987
2988     @classmethod
2989     def extract_id(cls, url):
2990         video_id = cls.get_temp_id(url)
2991         if not video_id:
2992             raise ExtractorError(f'Invalid URL: {url}')
2993         return video_id
2994
2995     def _extract_chapters_from_json(self, data, duration):
2996         chapter_list = traverse_obj(
2997             data, (
2998                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2999                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3000             ), expected_type=list)
3001
3002         return self._extract_chapters(
3003             chapter_list,
3004             chapter_time=lambda chapter: float_or_none(
3005                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3006             chapter_title=lambda chapter: traverse_obj(
3007                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3008             duration=duration)
3009
3010     def _extract_chapters_from_engagement_panel(self, data, duration):
3011         content_list = traverse_obj(
3012             data,
3013             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3014             expected_type=list, default=[])
3015         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3016         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3017
3018         return next(filter(None, (
3019             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3020                                    chapter_time, chapter_title, duration)
3021             for contents in content_list)), [])
3022
3023     def _extract_chapters_from_description(self, description, duration):
3024         return self._extract_chapters(
3025             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
3026             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
3027             duration=duration, strict=False)
3028
3029     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
3030         if not duration:
3031             return
3032         chapter_list = [{
3033             'start_time': chapter_time(chapter),
3034             'title': chapter_title(chapter),
3035         } for chapter in chapter_list or []]
3036         if not strict:
3037             chapter_list.sort(key=lambda c: c['start_time'] or 0)
3038
3039         chapters = [{'start_time': 0}]
3040         for idx, chapter in enumerate(chapter_list):
3041             if chapter['start_time'] is None:
3042                 self.report_warning(f'Incomplete chapter {idx}')
3043             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
3044                 chapters.append(chapter)
3045             elif chapter not in chapters:
3046                 self.report_warning(
3047                     f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
3048         return chapters[1:]
3049
3050     def _extract_comment(self, comment_renderer, parent=None):
3051         comment_id = comment_renderer.get('commentId')
3052         if not comment_id:
3053             return
3054
3055         text = self._get_text(comment_renderer, 'contentText')
3056
3057         # Timestamp is an estimate calculated from the current time and time_text
3058         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3059         timestamp = self._parse_time_text(time_text)
3060
3061         author = self._get_text(comment_renderer, 'authorText')
3062         author_id = try_get(comment_renderer,
3063                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3064
3065         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3066                                                        lambda x: x['likeCount']), str)) or 0
3067         author_thumbnail = try_get(comment_renderer,
3068                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3069
3070         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3071         is_favorited = 'creatorHeart' in (try_get(
3072             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3073         return {
3074             'id': comment_id,
3075             'text': text,
3076             'timestamp': timestamp,
3077             'time_text': time_text,
3078             'like_count': votes,
3079             'is_favorited': is_favorited,
3080             'author': author,
3081             'author_id': author_id,
3082             'author_thumbnail': author_thumbnail,
3083             'author_is_uploader': author_is_uploader,
3084             'parent': parent or 'root'
3085         }
3086
3087     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3088
3089         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3090
3091         def extract_header(contents):
3092             _continuation = None
3093             for content in contents:
3094                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3095                 expected_comment_count = self._get_count(
3096                     comments_header_renderer, 'countText', 'commentsCount')
3097
3098                 if expected_comment_count:
3099                     tracker['est_total'] = expected_comment_count
3100                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3101                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3102
3103                 sort_menu_item = try_get(
3104                     comments_header_renderer,
3105                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3106                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3107
3108                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3109                 if not _continuation:
3110                     continue
3111
3112                 sort_text = str_or_none(sort_menu_item.get('title'))
3113                 if not sort_text:
3114                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3115                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3116                 break
3117             return _continuation
3118
3119         def extract_thread(contents):
3120             if not parent:
3121                 tracker['current_page_thread'] = 0
3122             for content in contents:
3123                 if not parent and tracker['total_parent_comments'] >= max_parents:
3124                     yield
3125                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3126                 comment_renderer = get_first(
3127                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3128                     expected_type=dict, default={})
3129
3130                 comment = self._extract_comment(comment_renderer, parent)
3131                 if not comment:
3132                     continue
3133
3134                 tracker['running_total'] += 1
3135                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3136                 yield comment
3137
3138                 # Attempt to get the replies
3139                 comment_replies_renderer = try_get(
3140                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3141
3142                 if comment_replies_renderer:
3143                     tracker['current_page_thread'] += 1
3144                     comment_entries_iter = self._comment_entries(
3145                         comment_replies_renderer, ytcfg, video_id,
3146                         parent=comment.get('id'), tracker=tracker)
3147                     yield from itertools.islice(comment_entries_iter, min(
3148                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3149
3150         # Keeps track of counts across recursive calls
3151         if not tracker:
3152             tracker = dict(
3153                 running_total=0,
3154                 est_total=0,
3155                 current_page_thread=0,
3156                 total_parent_comments=0,
3157                 total_reply_comments=0)
3158
3159         # TODO: Deprecated
3160         # YouTube comments have a max depth of 2
3161         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3162         if max_depth:
3163             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3164                                                 'Set max replies in the max-comments extractor argument instead')
3165         if max_depth == 1 and parent:
3166             return
3167
3168         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3169             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3170
3171         continuation = self._extract_continuation(root_continuation_data)
3172
3173         response = None
3174         is_forced_continuation = False
3175         is_first_continuation = parent is None
3176         if is_first_continuation and not continuation:
3177             # Sometimes you can get comments by generating the continuation yourself,
3178             # even if YouTube initially reports them being disabled - e.g. stories comments.
3179             # Note: if the comment section is actually disabled, YouTube may return a response with
3180             # required check_get_keys missing. So we will disable that check initially in this case.
3181             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3182             is_forced_continuation = True
3183
3184         for page_num in itertools.count(0):
3185             if not continuation:
3186                 break
3187             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3188             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3189             if page_num == 0:
3190                 if is_first_continuation:
3191                     note_prefix = 'Downloading comment section API JSON'
3192                 else:
3193                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3194                         tracker['current_page_thread'], comment_prog_str)
3195             else:
3196                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3197                     '       ' if parent else '', ' replies' if parent else '',
3198                     page_num, comment_prog_str)
3199
3200             response = self._extract_response(
3201                 item_id=None, query=continuation,
3202                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3203                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3204             is_forced_continuation = False
3205             continuation_contents = traverse_obj(
3206                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3207
3208             continuation = None
3209             for continuation_section in continuation_contents:
3210                 continuation_items = traverse_obj(
3211                     continuation_section,
3212                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3213                     get_all=False, expected_type=list) or []
3214                 if is_first_continuation:
3215                     continuation = extract_header(continuation_items)
3216                     is_first_continuation = False
3217                     if continuation:
3218                         break
3219                     continue
3220
3221                 for entry in extract_thread(continuation_items):
3222                     if not entry:
3223                         return
3224                     yield entry
3225                 continuation = self._extract_continuation({'contents': continuation_items})
3226                 if continuation:
3227                     break
3228
3229         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3230         if message and not parent and tracker['running_total'] == 0:
3231             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3232
3233     @staticmethod
3234     def _generate_comment_continuation(video_id):
3235         """
3236         Generates initial comment section continuation token from given video id
3237         """
3238         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3239         return base64.b64encode(token.encode()).decode()
3240
3241     def _get_comments(self, ytcfg, video_id, contents, webpage):
3242         """Entry for comment extraction"""
3243         def _real_comment_extract(contents):
3244             renderer = next((
3245                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3246                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3247             yield from self._comment_entries(renderer, ytcfg, video_id)
3248
3249         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3250         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3251
3252     @staticmethod
3253     def _get_checkok_params():
3254         return {'contentCheckOk': True, 'racyCheckOk': True}
3255
3256     @classmethod
3257     def _generate_player_context(cls, sts=None):
3258         context = {
3259             'html5Preference': 'HTML5_PREF_WANTS',
3260         }
3261         if sts is not None:
3262             context['signatureTimestamp'] = sts
3263         return {
3264             'playbackContext': {
3265                 'contentPlaybackContext': context
3266             },
3267             **cls._get_checkok_params()
3268         }
3269
3270     @staticmethod
3271     def _is_agegated(player_response):
3272         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3273             return True
3274
3275         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3276         AGE_GATE_REASONS = (
3277             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3278             'age_verification_required', 'age_check_required',  # status
3279         )
3280         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3281
3282     @staticmethod
3283     def _is_unplayable(player_response):
3284         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3285
3286     _STORY_PLAYER_PARAMS = '8AEB'
3287
3288     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3289
3290         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3291         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3292         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3293         headers = self.generate_api_headers(
3294             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3295
3296         yt_query = {
3297             'videoId': video_id,
3298         }
3299         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3300             yt_query['params'] = self._STORY_PLAYER_PARAMS
3301
3302         yt_query.update(self._generate_player_context(sts))
3303         return self._extract_response(
3304             item_id=video_id, ep='player', query=yt_query,
3305             ytcfg=player_ytcfg, headers=headers, fatal=True,
3306             default_client=client,
3307             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3308         ) or None
3309
3310     def _get_requested_clients(self, url, smuggled_data):
3311         requested_clients = []
3312         default = ['android', 'web']
3313         allowed_clients = sorted(
3314             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3315             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3316         for client in self._configuration_arg('player_client'):
3317             if client in allowed_clients:
3318                 requested_clients.append(client)
3319             elif client == 'default':
3320                 requested_clients.extend(default)
3321             elif client == 'all':
3322                 requested_clients.extend(allowed_clients)
3323             else:
3324                 self.report_warning(f'Skipping unsupported client {client}')
3325         if not requested_clients:
3326             requested_clients = default
3327
3328         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3329             requested_clients.extend(
3330                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3331
3332         return orderedSet(requested_clients)
3333
3334     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3335         initial_pr = None
3336         if webpage:
3337             initial_pr = self._search_json(
3338                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3339
3340         all_clients = set(clients)
3341         clients = clients[::-1]
3342         prs = []
3343
3344         def append_client(*client_names):
3345             """ Append the first client name that exists but not already used """
3346             for client_name in client_names:
3347                 actual_client = _split_innertube_client(client_name)[0]
3348                 if actual_client in INNERTUBE_CLIENTS:
3349                     if actual_client not in all_clients:
3350                         clients.append(client_name)
3351                         all_clients.add(actual_client)
3352                         return
3353
3354         # Android player_response does not have microFormats which are needed for
3355         # extraction of some data. So we return the initial_pr with formats
3356         # stripped out even if not requested by the user
3357         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3358         if initial_pr:
3359             pr = dict(initial_pr)
3360             pr['streamingData'] = None
3361             prs.append(pr)
3362
3363         last_error = None
3364         tried_iframe_fallback = False
3365         player_url = None
3366         while clients:
3367             client, base_client, variant = _split_innertube_client(clients.pop())
3368             player_ytcfg = master_ytcfg if client == 'web' else {}
3369             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3370                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3371
3372             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3373             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3374             if 'js' in self._configuration_arg('player_skip'):
3375                 require_js_player = False
3376                 player_url = None
3377
3378             if not player_url and not tried_iframe_fallback and require_js_player:
3379                 player_url = self._download_player_url(video_id)
3380                 tried_iframe_fallback = True
3381
3382             try:
3383                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3384                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3385             except ExtractorError as e:
3386                 if last_error:
3387                     self.report_warning(last_error)
3388                 last_error = e
3389                 continue
3390
3391             if pr:
3392                 # YouTube may return a different video player response than expected.
3393                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3394                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3395                 if pr_video_id and pr_video_id != video_id:
3396                     self.report_warning(
3397                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3398                 else:
3399                     prs.append(pr)
3400
3401             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3402             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3403                 append_client(f'{base_client}_creator')
3404             elif self._is_agegated(pr):
3405                 if variant == 'tv_embedded':
3406                     append_client(f'{base_client}_embedded')
3407                 elif not variant:
3408                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3409
3410         if last_error:
3411             if not len(prs):
3412                 raise last_error
3413             self.report_warning(last_error)
3414         return prs, player_url
3415
3416     def _needs_live_processing(self, live_status, duration):
3417         if (live_status == 'is_live' and self.get_param('live_from_start')
3418                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3419             return live_status
3420
3421     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3422         itags, stream_ids = {}, []
3423         itag_qualities, res_qualities = {}, {0: None}
3424         q = qualities([
3425             # Normally tiny is the smallest video-only formats. But
3426             # audio-only formats with unknown quality may get tagged as tiny
3427             'tiny',
3428             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3429             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3430         ])
3431         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3432
3433         for fmt in streaming_formats:
3434             if fmt.get('targetDurationSec'):
3435                 continue
3436
3437             itag = str_or_none(fmt.get('itag'))
3438             audio_track = fmt.get('audioTrack') or {}
3439             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3440             if stream_id in stream_ids:
3441                 continue
3442
3443             quality = fmt.get('quality')
3444             height = int_or_none(fmt.get('height'))
3445             if quality == 'tiny' or not quality:
3446                 quality = fmt.get('audioQuality', '').lower() or quality
3447             # The 3gp format (17) in android client has a quality of "small",
3448             # but is actually worse than other formats
3449             if itag == '17':
3450                 quality = 'tiny'
3451             if quality:
3452                 if itag:
3453                     itag_qualities[itag] = quality
3454                 if height:
3455                     res_qualities[height] = quality
3456             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3457             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3458             # number of fragment that would subsequently requested with (`&sq=N`)
3459             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3460                 continue
3461
3462             fmt_url = fmt.get('url')
3463             if not fmt_url:
3464                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3465                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3466                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3467                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3468                     continue
3469                 try:
3470                     fmt_url += '&%s=%s' % (
3471                         traverse_obj(sc, ('sp', -1)) or 'signature',
3472                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3473                     )
3474                 except ExtractorError as e:
3475                     self.report_warning('Signature extraction failed: Some formats may be missing',
3476                                         video_id=video_id, only_once=True)
3477                     self.write_debug(e, only_once=True)
3478                     continue
3479
3480             query = parse_qs(fmt_url)
3481             throttled = False
3482             if query.get('n'):
3483                 try:
3484                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3485                     fmt_url = update_url_query(fmt_url, {
3486                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3487                     })
3488                 except ExtractorError as e:
3489                     phantomjs_hint = ''
3490                     if isinstance(e, JSInterpreter.Exception):
3491                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3492                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3493                     if player_url:
3494                         self.report_warning(
3495                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3496                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3497                         self.write_debug(e, only_once=True)
3498                     else:
3499                         self.report_warning(
3500                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3501                             video_id=video_id, only_once=True)
3502                     throttled = True
3503
3504             if itag:
3505                 itags[itag] = 'https'
3506                 stream_ids.append(stream_id)
3507
3508             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3509             language_preference = (
3510                 10 if audio_track.get('audioIsDefault') and 10
3511                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3512                 else -1)
3513             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3514             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3515             # Make sure to avoid false positives with small duration differences.
3516             # E.g. __2ABJjxzNo, ySuUZEjARPY
3517             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3518             if is_damaged:
3519                 self.report_warning(
3520                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3521             dct = {
3522                 'asr': int_or_none(fmt.get('audioSampleRate')),
3523                 'filesize': int_or_none(fmt.get('contentLength')),
3524                 'format_id': itag,
3525                 'format_note': join_nonempty(
3526                     '%s%s' % (audio_track.get('displayName') or '',
3527                               ' (default)' if language_preference > 0 else ''),
3528                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3529                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3530                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3531                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3532                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3533                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3534                 'fps': int_or_none(fmt.get('fps')) or None,
3535                 'audio_channels': fmt.get('audioChannels'),
3536                 'height': height,
3537                 'quality': q(quality),
3538                 'has_drm': bool(fmt.get('drmFamilies')),
3539                 'tbr': tbr,
3540                 'url': fmt_url,
3541                 'width': int_or_none(fmt.get('width')),
3542                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3543                                           'desc' if language_preference < -1 else ''),
3544                 'language_preference': language_preference,
3545                 # Strictly de-prioritize damaged and 3gp formats
3546                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3547             }
3548             mime_mobj = re.match(
3549                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3550             if mime_mobj:
3551                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3552                 dct.update(parse_codecs(mime_mobj.group(2)))
3553             no_audio = dct.get('acodec') == 'none'
3554             no_video = dct.get('vcodec') == 'none'
3555             if no_audio:
3556                 dct['vbr'] = tbr
3557             if no_video:
3558                 dct['abr'] = tbr
3559             if no_audio or no_video:
3560                 dct['downloader_options'] = {
3561                     # Youtube throttles chunks >~10M
3562                     'http_chunk_size': 10485760,
3563                 }
3564                 if dct.get('ext'):
3565                     dct['container'] = dct['ext'] + '_dash'
3566             yield dct
3567
3568         needs_live_processing = self._needs_live_processing(live_status, duration)
3569         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3570
3571         skip_manifests = set(self._configuration_arg('skip'))
3572         if (not self.get_param('youtube_include_hls_manifest', True)
3573                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3574                 or needs_live_processing and skip_bad_formats):
3575             skip_manifests.add('hls')
3576
3577         if not self.get_param('youtube_include_dash_manifest', True):
3578             skip_manifests.add('dash')
3579         if self._configuration_arg('include_live_dash'):
3580             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3581                                                 'Use include_incomplete_formats extractor argument instead')
3582         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3583             skip_manifests.add('dash')
3584
3585         def process_manifest_format(f, proto, itag):
3586             if itag in itags:
3587                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3588                     return False
3589                 itag = f'{itag}-{proto}'
3590             if itag:
3591                 f['format_id'] = itag
3592                 itags[itag] = proto
3593
3594             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3595             if f['quality'] == -1 and f.get('height'):
3596                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3597             return True
3598
3599         subtitles = {}
3600         for sd in streaming_data:
3601             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3602             if hls_manifest_url:
3603                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3604                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3605                 subtitles = self._merge_subtitles(subs, subtitles)
3606                 for f in fmts:
3607                     if process_manifest_format(f, 'hls', self._search_regex(
3608                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3609                         yield f
3610
3611             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3612             if dash_manifest_url:
3613                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3614                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3615                 for f in formats:
3616                     if process_manifest_format(f, 'dash', f['format_id']):
3617                         f['filesize'] = int_or_none(self._search_regex(
3618                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3619                         if needs_live_processing:
3620                             f['is_from_start'] = True
3621
3622                         yield f
3623         yield subtitles
3624
3625     def _extract_storyboard(self, player_responses, duration):
3626         spec = get_first(
3627             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3628         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3629         if not base_url:
3630             return
3631         L = len(spec) - 1
3632         for i, args in enumerate(spec):
3633             args = args.split('#')
3634             counts = list(map(int_or_none, args[:5]))
3635             if len(args) != 8 or not all(counts):
3636                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3637                 continue
3638             width, height, frame_count, cols, rows = counts
3639             N, sigh = args[6:]
3640
3641             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3642             fragment_count = frame_count / (cols * rows)
3643             fragment_duration = duration / fragment_count
3644             yield {
3645                 'format_id': f'sb{i}',
3646                 'format_note': 'storyboard',
3647                 'ext': 'mhtml',
3648                 'protocol': 'mhtml',
3649                 'acodec': 'none',
3650                 'vcodec': 'none',
3651                 'url': url,
3652                 'width': width,
3653                 'height': height,
3654                 'fps': frame_count / duration,
3655                 'rows': rows,
3656                 'columns': cols,
3657                 'fragments': [{
3658                     'url': url.replace('$M', str(j)),
3659                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3660                 } for j in range(math.ceil(fragment_count))],
3661             }
3662
3663     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3664         webpage = None
3665         if 'webpage' not in self._configuration_arg('player_skip'):
3666             query = {'bpctr': '9999999999', 'has_verified': '1'}
3667             if smuggled_data.get('is_story'):
3668                 query['pp'] = self._STORY_PLAYER_PARAMS
3669             webpage = self._download_webpage(
3670                 webpage_url, video_id, fatal=False, query=query)
3671
3672         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3673
3674         player_responses, player_url = self._extract_player_responses(
3675             self._get_requested_clients(url, smuggled_data),
3676             video_id, webpage, master_ytcfg, smuggled_data)
3677
3678         return webpage, master_ytcfg, player_responses, player_url
3679
3680     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3681         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3682         is_live = get_first(video_details, 'isLive')
3683         if is_live is None:
3684             is_live = get_first(live_broadcast_details, 'isLiveNow')
3685         live_content = get_first(video_details, 'isLiveContent')
3686         is_upcoming = get_first(video_details, 'isUpcoming')
3687         post_live = get_first(video_details, 'isPostLiveDvr')
3688         live_status = ('post_live' if post_live
3689                        else 'is_live' if is_live
3690                        else 'is_upcoming' if is_upcoming
3691                        else 'was_live' if live_content
3692                        else 'not_live' if False in (is_live, live_content)
3693                        else None)
3694         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3695         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3696
3697         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3698
3699     def _real_extract(self, url):
3700         url, smuggled_data = unsmuggle_url(url, {})
3701         video_id = self._match_id(url)
3702
3703         base_url = self.http_scheme() + '//www.youtube.com/'
3704         webpage_url = base_url + 'watch?v=' + video_id
3705
3706         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3707
3708         playability_statuses = traverse_obj(
3709             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3710
3711         trailer_video_id = get_first(
3712             playability_statuses,
3713             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3714             expected_type=str)
3715         if trailer_video_id:
3716             return self.url_result(
3717                 trailer_video_id, self.ie_key(), trailer_video_id)
3718
3719         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3720                        if webpage else (lambda x: None))
3721
3722         video_details = traverse_obj(
3723             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3724         microformats = traverse_obj(
3725             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3726             expected_type=dict, default=[])
3727
3728         translated_title = self._get_text(microformats, (..., 'title'))
3729         video_title = (self._preferred_lang and translated_title
3730                        or get_first(video_details, 'title')  # primary
3731                        or translated_title
3732                        or search_meta(['og:title', 'twitter:title', 'title']))
3733         translated_description = self._get_text(microformats, (..., 'description'))
3734         original_description = get_first(video_details, 'shortDescription')
3735         video_description = (
3736             self._preferred_lang and translated_description
3737             # If original description is blank, it will be an empty string.
3738             # Do not prefer translated description in this case.
3739             or original_description if original_description is not None else translated_description)
3740
3741         multifeed_metadata_list = get_first(
3742             player_responses,
3743             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3744             expected_type=str)
3745         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3746             if self.get_param('noplaylist'):
3747                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3748             else:
3749                 entries = []
3750                 feed_ids = []
3751                 for feed in multifeed_metadata_list.split(','):
3752                     # Unquote should take place before split on comma (,) since textual
3753                     # fields may contain comma as well (see
3754                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3755                     feed_data = urllib.parse.parse_qs(
3756                         urllib.parse.unquote_plus(feed))
3757
3758                     def feed_entry(name):
3759                         return try_get(
3760                             feed_data, lambda x: x[name][0], str)
3761
3762                     feed_id = feed_entry('id')
3763                     if not feed_id:
3764                         continue
3765                     feed_title = feed_entry('title')
3766                     title = video_title
3767                     if feed_title:
3768                         title += ' (%s)' % feed_title
3769                     entries.append({
3770                         '_type': 'url_transparent',
3771                         'ie_key': 'Youtube',
3772                         'url': smuggle_url(
3773                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3774                             {'force_singlefeed': True}),
3775                         'title': title,
3776                     })
3777                     feed_ids.append(feed_id)
3778                 self.to_screen(
3779                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3780                     % (', '.join(feed_ids), video_id))
3781                 return self.playlist_result(
3782                     entries, video_id, video_title, video_description)
3783
3784         duration = int_or_none(
3785             get_first(video_details, 'lengthSeconds')
3786             or get_first(microformats, 'lengthSeconds')
3787             or parse_duration(search_meta('duration'))) or None
3788
3789         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
3790             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
3791         if live_status == 'post_live':
3792             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
3793
3794         if not formats:
3795             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3796                 self.report_drm(video_id)
3797             pemr = get_first(
3798                 playability_statuses,
3799                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3800             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3801             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3802             if subreason:
3803                 if subreason == 'The uploader has not made this video available in your country.':
3804                     countries = get_first(microformats, 'availableCountries')
3805                     if not countries:
3806                         regions_allowed = search_meta('regionsAllowed')
3807                         countries = regions_allowed.split(',') if regions_allowed else None
3808                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3809                 reason += f'. {subreason}'
3810             if reason:
3811                 self.raise_no_formats(reason, expected=True)
3812
3813         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3814         if not keywords and webpage:
3815             keywords = [
3816                 unescapeHTML(m.group('content'))
3817                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3818         for keyword in keywords:
3819             if keyword.startswith('yt:stretch='):
3820                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3821                 if mobj:
3822                     # NB: float is intentional for forcing float division
3823                     w, h = (float(v) for v in mobj.groups())
3824                     if w > 0 and h > 0:
3825                         ratio = w / h
3826                         for f in formats:
3827                             if f.get('vcodec') != 'none':
3828                                 f['stretched_ratio'] = ratio
3829                         break
3830         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3831         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3832         if thumbnail_url:
3833             thumbnails.append({
3834                 'url': thumbnail_url,
3835             })
3836         original_thumbnails = thumbnails.copy()
3837
3838         # The best resolution thumbnails sometimes does not appear in the webpage
3839         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3840         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3841         thumbnail_names = [
3842             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3843             # in resolution, these are not the custom thumbnail. So de-prioritize them
3844             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3845             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3846         ]
3847         n_thumbnail_names = len(thumbnail_names)
3848         thumbnails.extend({
3849             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3850                 video_id=video_id, name=name, ext=ext,
3851                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
3852         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3853         for thumb in thumbnails:
3854             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3855             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3856         self._remove_duplicate_formats(thumbnails)
3857         self._downloader._sort_thumbnails(original_thumbnails)
3858
3859         category = get_first(microformats, 'category') or search_meta('genre')
3860         channel_id = str_or_none(
3861             get_first(video_details, 'channelId')
3862             or get_first(microformats, 'externalChannelId')
3863             or search_meta('channelId'))
3864         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3865
3866         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3867         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3868         if not duration and live_end_time and live_start_time:
3869             duration = live_end_time - live_start_time
3870
3871         needs_live_processing = self._needs_live_processing(live_status, duration)
3872
3873         def is_bad_format(fmt):
3874             if needs_live_processing and not fmt.get('is_from_start'):
3875                 return True
3876             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
3877                     and fmt.get('protocol') == 'http_dash_segments'):
3878                 return True
3879
3880         for fmt in filter(is_bad_format, formats):
3881             fmt['preference'] = (fmt.get('preference') or -1) - 10
3882             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
3883
3884         if needs_live_processing:
3885             self._prepare_live_from_start_formats(
3886                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
3887
3888         formats.extend(self._extract_storyboard(player_responses, duration))
3889
3890         # source_preference is lower for throttled/potentially damaged formats
3891         self._sort_formats(formats, (
3892             'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto'))
3893
3894         info = {
3895             'id': video_id,
3896             'title': video_title,
3897             'formats': formats,
3898             'thumbnails': thumbnails,
3899             # The best thumbnail that we are sure exists. Prevents unnecessary
3900             # URL checking if user don't care about getting the best possible thumbnail
3901             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3902             'description': video_description,
3903             'uploader': get_first(video_details, 'author'),
3904             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3905             'uploader_url': owner_profile_url,
3906             'channel_id': channel_id,
3907             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3908             'duration': duration,
3909             'view_count': int_or_none(
3910                 get_first((video_details, microformats), (..., 'viewCount'))
3911                 or search_meta('interactionCount')),
3912             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3913             'age_limit': 18 if (
3914                 get_first(microformats, 'isFamilySafe') is False
3915                 or search_meta('isFamilyFriendly') == 'false'
3916                 or search_meta('og:restrictions:age') == '18+') else 0,
3917             'webpage_url': webpage_url,
3918             'categories': [category] if category else None,
3919             'tags': keywords,
3920             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3921             'live_status': live_status,
3922             'release_timestamp': live_start_time,
3923         }
3924
3925         subtitles = {}
3926         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3927         if pctr:
3928             def get_lang_code(track):
3929                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3930                         or track.get('languageCode'))
3931
3932             # Converted into dicts to remove duplicates
3933             captions = {
3934                 get_lang_code(sub): sub
3935                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3936             translation_languages = {
3937                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3938                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3939
3940             def process_language(container, base_url, lang_code, sub_name, query):
3941                 lang_subs = container.setdefault(lang_code, [])
3942                 for fmt in self._SUBTITLE_FORMATS:
3943                     query.update({
3944                         'fmt': fmt,
3945                     })
3946                     lang_subs.append({
3947                         'ext': fmt,
3948                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3949                         'name': sub_name,
3950                     })
3951
3952             # NB: Constructing the full subtitle dictionary is slow
3953             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
3954                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
3955             for lang_code, caption_track in captions.items():
3956                 base_url = caption_track.get('baseUrl')
3957                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3958                 if not base_url:
3959                     continue
3960                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3961                 if caption_track.get('kind') != 'asr':
3962                     if not lang_code:
3963                         continue
3964                     process_language(
3965                         subtitles, base_url, lang_code, lang_name, {})
3966                     if not caption_track.get('isTranslatable'):
3967                         continue
3968                 for trans_code, trans_name in translation_languages.items():
3969                     if not trans_code:
3970                         continue
3971                     orig_trans_code = trans_code
3972                     if caption_track.get('kind') != 'asr':
3973                         if not get_translated_subs:
3974                             continue
3975                         trans_code += f'-{lang_code}'
3976                         trans_name += format_field(lang_name, None, ' from %s')
3977                     # Add an "-orig" label to the original language so that it can be distinguished.
3978                     # The subs are returned without "-orig" as well for compatibility
3979                     if lang_code == f'a-{orig_trans_code}':
3980                         process_language(
3981                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3982                     # Setting tlang=lang returns damaged subtitles.
3983                     process_language(automatic_captions, base_url, trans_code, trans_name,
3984                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3985
3986         info['automatic_captions'] = automatic_captions
3987         info['subtitles'] = subtitles
3988
3989         parsed_url = urllib.parse.urlparse(url)
3990         for component in [parsed_url.fragment, parsed_url.query]:
3991             query = urllib.parse.parse_qs(component)
3992             for k, v in query.items():
3993                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3994                     d_k += '_time'
3995                     if d_k not in info and k in s_ks:
3996                         info[d_k] = parse_duration(query[k][0])
3997
3998         # Youtube Music Auto-generated description
3999         if video_description:
4000             mobj = re.search(
4001                 r'''(?xs)
4002                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4003                     (?P<album>[^\n]+)
4004                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4005                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4006                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4007                     .+\nAuto-generated\ by\ YouTube\.\s*$
4008                 ''', video_description)
4009             if mobj:
4010                 release_year = mobj.group('release_year')
4011                 release_date = mobj.group('release_date')
4012                 if release_date:
4013                     release_date = release_date.replace('-', '')
4014                     if not release_year:
4015                         release_year = release_date[:4]
4016                 info.update({
4017                     'album': mobj.group('album'.strip()),
4018                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4019                     'track': mobj.group('track').strip(),
4020                     'release_date': release_date,
4021                     'release_year': int_or_none(release_year),
4022                 })
4023
4024         initial_data = None
4025         if webpage:
4026             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4027         if not initial_data:
4028             query = {'videoId': video_id}
4029             query.update(self._get_checkok_params())
4030             initial_data = self._extract_response(
4031                 item_id=video_id, ep='next', fatal=False,
4032                 ytcfg=master_ytcfg, query=query,
4033                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4034                 note='Downloading initial data API JSON')
4035
4036         info['comment_count'] = traverse_obj(initial_data, (
4037             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4038             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
4039         ), (
4040             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4041             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
4042         ), expected_type=int_or_none, get_all=False)
4043
4044         try:  # This will error if there is no livechat
4045             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4046         except (KeyError, IndexError, TypeError):
4047             pass
4048         else:
4049             info.setdefault('subtitles', {})['live_chat'] = [{
4050                 # url is needed to set cookies
4051                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4052                 'video_id': video_id,
4053                 'ext': 'json',
4054                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4055                              else 'youtube_live_chat_replay'),
4056             }]
4057
4058         if initial_data:
4059             info['chapters'] = (
4060                 self._extract_chapters_from_json(initial_data, duration)
4061                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4062                 or self._extract_chapters_from_description(video_description, duration)
4063                 or None)
4064
4065         contents = traverse_obj(
4066             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4067             expected_type=list, default=[])
4068
4069         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4070         if vpir:
4071             stl = vpir.get('superTitleLink')
4072             if stl:
4073                 stl = self._get_text(stl)
4074                 if try_get(
4075                         vpir,
4076                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4077                     info['location'] = stl
4078                 else:
4079                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4080                     if mobj:
4081                         info.update({
4082                             'series': mobj.group(1),
4083                             'season_number': int(mobj.group(2)),
4084                             'episode_number': int(mobj.group(3)),
4085                         })
4086             for tlb in (try_get(
4087                     vpir,
4088                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4089                     list) or []):
4090                 tbrs = variadic(
4091                     traverse_obj(
4092                         tlb, 'toggleButtonRenderer',
4093                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer'),
4094                         default=[]))
4095                 for tbr in tbrs:
4096                     for getter, regex in [(
4097                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4098                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4099                                 lambda x: x['accessibility'],
4100                                 lambda x: x['accessibilityData']['accessibilityData'],
4101                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4102                         label = (try_get(tbr, getter, dict) or {}).get('label')
4103                         if label:
4104                             mobj = re.match(regex, label)
4105                             if mobj:
4106                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4107                                 break
4108             sbr_tooltip = try_get(
4109                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4110             if sbr_tooltip:
4111                 like_count, dislike_count = sbr_tooltip.split(' / ')
4112                 info.update({
4113                     'like_count': str_to_int(like_count),
4114                     'dislike_count': str_to_int(dislike_count),
4115                 })
4116             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4117             if vcr:
4118                 vc = self._get_count(vcr, 'viewCount')
4119                 # Upcoming premieres with waiting count are treated as live here
4120                 if vcr.get('isLive'):
4121                     info['concurrent_view_count'] = vc
4122                 elif info.get('view_count') is None:
4123                     info['view_count'] = vc
4124
4125         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4126         if vsir:
4127             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4128             info.update({
4129                 'channel': self._get_text(vor, 'title'),
4130                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4131
4132             rows = try_get(
4133                 vsir,
4134                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4135                 list) or []
4136             multiple_songs = False
4137             for row in rows:
4138                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4139                     multiple_songs = True
4140                     break
4141             for row in rows:
4142                 mrr = row.get('metadataRowRenderer') or {}
4143                 mrr_title = mrr.get('title')
4144                 if not mrr_title:
4145                     continue
4146                 mrr_title = self._get_text(mrr, 'title')
4147                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4148                 if mrr_title == 'License':
4149                     info['license'] = mrr_contents_text
4150                 elif not multiple_songs:
4151                     if mrr_title == 'Album':
4152                         info['album'] = mrr_contents_text
4153                     elif mrr_title == 'Artist':
4154                         info['artist'] = mrr_contents_text
4155                     elif mrr_title == 'Song':
4156                         info['track'] = mrr_contents_text
4157
4158         fallbacks = {
4159             'channel': 'uploader',
4160             'channel_id': 'uploader_id',
4161             'channel_url': 'uploader_url',
4162         }
4163
4164         # The upload date for scheduled, live and past live streams / premieres in microformats
4165         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4166         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4167         upload_date = (
4168             unified_strdate(get_first(microformats, 'uploadDate'))
4169             or unified_strdate(search_meta('uploadDate')))
4170         if not upload_date or (
4171             live_status in ('not_live', None)
4172             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4173         ):
4174             upload_date = strftime_or_none(
4175                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4176         info['upload_date'] = upload_date
4177
4178         for to, frm in fallbacks.items():
4179             if not info.get(to):
4180                 info[to] = info.get(frm)
4181
4182         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4183             v = info.get(s_k)
4184             if v:
4185                 info[d_k] = v
4186
4187         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4188
4189         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4190                       or get_first(video_details, 'isPrivate', expected_type=bool))
4191
4192         info['availability'] = (
4193             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4194             else self._availability(
4195                 is_private=is_private,
4196                 needs_premium=(
4197                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4198                     or False if initial_data and is_private is not None else None),
4199                 needs_subscription=(
4200                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4201                     or False if initial_data and is_private is not None else None),
4202                 needs_auth=info['age_limit'] >= 18,
4203                 is_unlisted=None if is_private is None else (
4204                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4205                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4206
4207         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4208
4209         self.mark_watched(video_id, player_responses)
4210
4211         return info
4212
4213
4214 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4215
4216     @staticmethod
4217     def passthrough_smuggled_data(func):
4218         def _smuggle(entries, smuggled_data):
4219             for entry in entries:
4220                 # TODO: Convert URL to music.youtube instead.
4221                 # Do we need to passthrough any other smuggled_data?
4222                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
4223                 yield entry
4224
4225         @functools.wraps(func)
4226         def wrapper(self, url):
4227             url, smuggled_data = unsmuggle_url(url, {})
4228             if self.is_music_url(url):
4229                 smuggled_data['is_music_url'] = True
4230             info_dict = func(self, url, smuggled_data)
4231             if smuggled_data and info_dict.get('entries'):
4232                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
4233             return info_dict
4234         return wrapper
4235
4236     def _extract_channel_id(self, webpage):
4237         channel_id = self._html_search_meta(
4238             'channelId', webpage, 'channel id', default=None)
4239         if channel_id:
4240             return channel_id
4241         channel_url = self._html_search_meta(
4242             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
4243              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
4244              'twitter:app:url:googleplay'), webpage, 'channel url')
4245         return self._search_regex(
4246             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
4247             channel_url, 'channel id')
4248
4249     @staticmethod
4250     def _extract_basic_item_renderer(item):
4251         # Modified from _extract_grid_item_renderer
4252         known_basic_renderers = (
4253             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4254         )
4255         for key, renderer in item.items():
4256             if not isinstance(renderer, dict):
4257                 continue
4258             elif key in known_basic_renderers:
4259                 return renderer
4260             elif key.startswith('grid') and key.endswith('Renderer'):
4261                 return renderer
4262
4263     def _grid_entries(self, grid_renderer):
4264         for item in grid_renderer['items']:
4265             if not isinstance(item, dict):
4266                 continue
4267             renderer = self._extract_basic_item_renderer(item)
4268             if not isinstance(renderer, dict):
4269                 continue
4270             title = self._get_text(renderer, 'title')
4271
4272             # playlist
4273             playlist_id = renderer.get('playlistId')
4274             if playlist_id:
4275                 yield self.url_result(
4276                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4277                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4278                     video_title=title)
4279                 continue
4280             # video
4281             video_id = renderer.get('videoId')
4282             if video_id:
4283                 yield self._extract_video(renderer)
4284                 continue
4285             # channel
4286             channel_id = renderer.get('channelId')
4287             if channel_id:
4288                 yield self.url_result(
4289                     'https://www.youtube.com/channel/%s' % channel_id,
4290                     ie=YoutubeTabIE.ie_key(), video_title=title)
4291                 continue
4292             # generic endpoint URL support
4293             ep_url = urljoin('https://www.youtube.com/', try_get(
4294                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4295                 str))
4296             if ep_url:
4297                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4298                     if ie.suitable(ep_url):
4299                         yield self.url_result(
4300                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4301                         break
4302
4303     def _music_reponsive_list_entry(self, renderer):
4304         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4305         if video_id:
4306             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4307                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4308         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4309         if playlist_id:
4310             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4311             if video_id:
4312                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4313                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4314             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4315                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4316         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4317         if browse_id:
4318             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4319                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4320
4321     def _shelf_entries_from_content(self, shelf_renderer):
4322         content = shelf_renderer.get('content')
4323         if not isinstance(content, dict):
4324             return
4325         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4326         if renderer:
4327             # TODO: add support for nested playlists so each shelf is processed
4328             # as separate playlist
4329             # TODO: this includes only first N items
4330             yield from self._grid_entries(renderer)
4331         renderer = content.get('horizontalListRenderer')
4332         if renderer:
4333             # TODO
4334             pass
4335
4336     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4337         ep = try_get(
4338             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4339             str)
4340         shelf_url = urljoin('https://www.youtube.com', ep)
4341         if shelf_url:
4342             # Skipping links to another channels, note that checking for
4343             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4344             # will not work
4345             if skip_channels and '/channels?' in shelf_url:
4346                 return
4347             title = self._get_text(shelf_renderer, 'title')
4348             yield self.url_result(shelf_url, video_title=title)
4349         # Shelf may not contain shelf URL, fallback to extraction from content
4350         yield from self._shelf_entries_from_content(shelf_renderer)
4351
4352     def _playlist_entries(self, video_list_renderer):
4353         for content in video_list_renderer['contents']:
4354             if not isinstance(content, dict):
4355                 continue
4356             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4357             if not isinstance(renderer, dict):
4358                 continue
4359             video_id = renderer.get('videoId')
4360             if not video_id:
4361                 continue
4362             yield self._extract_video(renderer)
4363
4364     def _rich_entries(self, rich_grid_renderer):
4365         renderer = traverse_obj(
4366             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4367         video_id = renderer.get('videoId')
4368         if not video_id:
4369             return
4370         yield self._extract_video(renderer)
4371
4372     def _video_entry(self, video_renderer):
4373         video_id = video_renderer.get('videoId')
4374         if video_id:
4375             return self._extract_video(video_renderer)
4376
4377     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4378         url = urljoin('https://youtube.com', traverse_obj(
4379             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4380         if url:
4381             return self.url_result(
4382                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4383
4384     def _post_thread_entries(self, post_thread_renderer):
4385         post_renderer = try_get(
4386             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4387         if not post_renderer:
4388             return
4389         # video attachment
4390         video_renderer = try_get(
4391             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4392         video_id = video_renderer.get('videoId')
4393         if video_id:
4394             entry = self._extract_video(video_renderer)
4395             if entry:
4396                 yield entry
4397         # playlist attachment
4398         playlist_id = try_get(
4399             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4400         if playlist_id:
4401             yield self.url_result(
4402                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4403                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4404         # inline video links
4405         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4406         for run in runs:
4407             if not isinstance(run, dict):
4408                 continue
4409             ep_url = try_get(
4410                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4411             if not ep_url:
4412                 continue
4413             if not YoutubeIE.suitable(ep_url):
4414                 continue
4415             ep_video_id = YoutubeIE._match_id(ep_url)
4416             if video_id == ep_video_id:
4417                 continue
4418             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4419
4420     def _post_thread_continuation_entries(self, post_thread_continuation):
4421         contents = post_thread_continuation.get('contents')
4422         if not isinstance(contents, list):
4423             return
4424         for content in contents:
4425             renderer = content.get('backstagePostThreadRenderer')
4426             if isinstance(renderer, dict):
4427                 yield from self._post_thread_entries(renderer)
4428                 continue
4429             renderer = content.get('videoRenderer')
4430             if isinstance(renderer, dict):
4431                 yield self._video_entry(renderer)
4432
4433     r''' # unused
4434     def _rich_grid_entries(self, contents):
4435         for content in contents:
4436             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4437             if video_renderer:
4438                 entry = self._video_entry(video_renderer)
4439                 if entry:
4440                     yield entry
4441     '''
4442
4443     def _report_history_entries(self, renderer):
4444         for url in traverse_obj(renderer, (
4445                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4446                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4447                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4448             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4449
4450     def _extract_entries(self, parent_renderer, continuation_list):
4451         # continuation_list is modified in-place with continuation_list = [continuation_token]
4452         continuation_list[:] = [None]
4453         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4454         for content in contents:
4455             if not isinstance(content, dict):
4456                 continue
4457             is_renderer = traverse_obj(
4458                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4459                 expected_type=dict)
4460             if not is_renderer:
4461                 if content.get('richItemRenderer'):
4462                     for entry in self._rich_entries(content['richItemRenderer']):
4463                         yield entry
4464                     continuation_list[0] = self._extract_continuation(parent_renderer)
4465                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4466                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4467                     yield from self._report_history_entries(table)
4468                     continuation_list[0] = self._extract_continuation(table)
4469                 continue
4470
4471             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4472             for isr_content in isr_contents:
4473                 if not isinstance(isr_content, dict):
4474                     continue
4475
4476                 known_renderers = {
4477                     'playlistVideoListRenderer': self._playlist_entries,
4478                     'gridRenderer': self._grid_entries,
4479                     'reelShelfRenderer': self._grid_entries,
4480                     'shelfRenderer': self._shelf_entries,
4481                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4482                     'backstagePostThreadRenderer': self._post_thread_entries,
4483                     'videoRenderer': lambda x: [self._video_entry(x)],
4484                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4485                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4486                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4487                 }
4488                 for key, renderer in isr_content.items():
4489                     if key not in known_renderers:
4490                         continue
4491                     for entry in known_renderers[key](renderer):
4492                         if entry:
4493                             yield entry
4494                     continuation_list[0] = self._extract_continuation(renderer)
4495                     break
4496
4497             if not continuation_list[0]:
4498                 continuation_list[0] = self._extract_continuation(is_renderer)
4499
4500         if not continuation_list[0]:
4501             continuation_list[0] = self._extract_continuation(parent_renderer)
4502
4503     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4504         continuation_list = [None]
4505         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4506         tab_content = try_get(tab, lambda x: x['content'], dict)
4507         if not tab_content:
4508             return
4509         parent_renderer = (
4510             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4511             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4512         yield from extract_entries(parent_renderer)
4513         continuation = continuation_list[0]
4514
4515         for page_num in itertools.count(1):
4516             if not continuation:
4517                 break
4518             headers = self.generate_api_headers(
4519                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4520             response = self._extract_response(
4521                 item_id=f'{item_id} page {page_num}',
4522                 query=continuation, headers=headers, ytcfg=ytcfg,
4523                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4524
4525             if not response:
4526                 break
4527             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4528             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4529             visitor_data = self._extract_visitor_data(response) or visitor_data
4530
4531             known_renderers = {
4532                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4533                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4534                 'gridVideoRenderer': (self._grid_entries, 'items'),
4535                 'gridChannelRenderer': (self._grid_entries, 'items'),
4536                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4537                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4538                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4539                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4540                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4541                 'playlistVideoListContinuation': (self._playlist_entries, None),
4542                 'gridContinuation': (self._grid_entries, None),
4543                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4544                 'sectionListContinuation': (extract_entries, None),  # for feeds
4545             }
4546
4547             continuation_items = traverse_obj(response, (
4548                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4549                 'appendContinuationItemsAction', 'continuationItems'
4550             ), 'continuationContents', get_all=False)
4551             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4552
4553             video_items_renderer = None
4554             for key in continuation_item.keys():
4555                 if key not in known_renderers:
4556                     continue
4557                 func, parent_key = known_renderers[key]
4558                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4559                 continuation_list = [None]
4560                 yield from func(video_items_renderer)
4561                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4562
4563             if not video_items_renderer:
4564                 break
4565
4566     @staticmethod
4567     def _extract_selected_tab(tabs, fatal=True):
4568         for tab in tabs:
4569             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4570             if renderer.get('selected') is True:
4571                 return renderer
4572         else:
4573             if fatal:
4574                 raise ExtractorError('Unable to find selected tab')
4575
4576     def _extract_uploader(self, data):
4577         uploader = {}
4578         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4579         owner = try_get(
4580             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4581         if owner:
4582             owner_text = owner.get('text')
4583             uploader['uploader'] = self._search_regex(
4584                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4585             uploader['uploader_id'] = try_get(
4586                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], str)
4587             uploader['uploader_url'] = urljoin(
4588                 'https://www.youtube.com/',
4589                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], str))
4590         return filter_dict(uploader)
4591
4592     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4593         playlist_id = title = description = channel_url = channel_name = channel_id = None
4594         tags = []
4595
4596         selected_tab = self._extract_selected_tab(tabs)
4597         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4598         renderer = try_get(
4599             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4600         if renderer:
4601             channel_name = renderer.get('title')
4602             channel_url = renderer.get('channelUrl')
4603             channel_id = renderer.get('externalId')
4604         else:
4605             renderer = try_get(
4606                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4607
4608         if renderer:
4609             title = renderer.get('title')
4610             description = renderer.get('description', '')
4611             playlist_id = channel_id
4612             tags = renderer.get('keywords', '').split()
4613
4614         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4615         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4616         def _get_uncropped(url):
4617             return url_or_none((url or '').split('=')[0] + '=s0')
4618
4619         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4620         if avatar_thumbnails:
4621             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4622             if uncropped_avatar:
4623                 avatar_thumbnails.append({
4624                     'url': uncropped_avatar,
4625                     'id': 'avatar_uncropped',
4626                     'preference': 1
4627                 })
4628
4629         channel_banners = self._extract_thumbnails(
4630             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4631         for banner in channel_banners:
4632             banner['preference'] = -10
4633
4634         if channel_banners:
4635             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4636             if uncropped_banner:
4637                 channel_banners.append({
4638                     'url': uncropped_banner,
4639                     'id': 'banner_uncropped',
4640                     'preference': -5
4641                 })
4642
4643         primary_thumbnails = self._extract_thumbnails(
4644             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4645
4646         if playlist_id is None:
4647             playlist_id = item_id
4648
4649         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4650         last_updated_unix = self._parse_time_text(self._get_text(playlist_stats, 2))
4651         if title is None:
4652             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4653         title += format_field(selected_tab, 'title', ' - %s')
4654         title += format_field(selected_tab, 'expandedText', ' - %s')
4655
4656         metadata = {
4657             'playlist_id': playlist_id,
4658             'playlist_title': title,
4659             'playlist_description': description,
4660             'uploader': channel_name,
4661             'uploader_id': channel_id,
4662             'uploader_url': channel_url,
4663             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4664             'tags': tags,
4665             'view_count': self._get_count(playlist_stats, 1),
4666             'availability': self._extract_availability(data),
4667             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4668             'playlist_count': self._get_count(playlist_stats, 0),
4669             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4670         }
4671         if not channel_id:
4672             metadata.update(self._extract_uploader(data))
4673         metadata.update({
4674             'channel': metadata['uploader'],
4675             'channel_id': metadata['uploader_id'],
4676             'channel_url': metadata['uploader_url']})
4677         return self.playlist_result(
4678             self._entries(
4679                 selected_tab, playlist_id, ytcfg,
4680                 self._extract_account_syncid(ytcfg, data),
4681                 self._extract_visitor_data(data, ytcfg)),
4682             **metadata)
4683
4684     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4685         first_id = last_id = response = None
4686         for page_num in itertools.count(1):
4687             videos = list(self._playlist_entries(playlist))
4688             if not videos:
4689                 return
4690             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4691             if start >= len(videos):
4692                 return
4693             yield from videos[start:]
4694             first_id = first_id or videos[0]['id']
4695             last_id = videos[-1]['id']
4696             watch_endpoint = try_get(
4697                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4698             headers = self.generate_api_headers(
4699                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4700                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4701             query = {
4702                 'playlistId': playlist_id,
4703                 'videoId': watch_endpoint.get('videoId') or last_id,
4704                 'index': watch_endpoint.get('index') or len(videos),
4705                 'params': watch_endpoint.get('params') or 'OAE%3D'
4706             }
4707             response = self._extract_response(
4708                 item_id='%s page %d' % (playlist_id, page_num),
4709                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4710                 check_get_keys='contents'
4711             )
4712             playlist = try_get(
4713                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4714
4715     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4716         title = playlist.get('title') or try_get(
4717             data, lambda x: x['titleText']['simpleText'], str)
4718         playlist_id = playlist.get('playlistId') or item_id
4719
4720         # Delegating everything except mix playlists to regular tab-based playlist URL
4721         playlist_url = urljoin(url, try_get(
4722             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4723             str))
4724
4725         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4726         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4727         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4728
4729         if playlist_url and playlist_url != url and not is_known_unviewable:
4730             return self.url_result(
4731                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4732                 video_title=title)
4733
4734         return self.playlist_result(
4735             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4736             playlist_id=playlist_id, playlist_title=title)
4737
4738     def _extract_availability(self, data):
4739         """
4740         Gets the availability of a given playlist/tab.
4741         Note: Unless YouTube tells us explicitly, we do not assume it is public
4742         @param data: response
4743         """
4744         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4745
4746         player_header_privacy = traverse_obj(
4747             data, ('header', 'playlistHeaderRenderer', 'privacy'), expected_type=str)
4748
4749         badges = self._extract_badges(renderer)
4750
4751         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4752         privacy_setting_icon = traverse_obj(
4753             renderer, (
4754                 'privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
4755                 lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
4756             get_all=False, expected_type=str)
4757
4758         return (
4759             'public' if (
4760                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4761                 or player_header_privacy == 'PUBLIC'
4762                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
4763             else self._availability(
4764                 is_private=(
4765                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4766                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
4767                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
4768                 is_unlisted=(
4769                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4770                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
4771                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None else None),
4772                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
4773                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
4774                 needs_auth=False))
4775
4776     @staticmethod
4777     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4778         sidebar_renderer = try_get(
4779             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4780         for item in sidebar_renderer:
4781             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4782             if renderer:
4783                 return renderer
4784
4785     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4786         """
4787         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4788         """
4789         browse_id = params = None
4790         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4791         if not renderer:
4792             return
4793         menu_renderer = try_get(
4794             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4795         for menu_item in menu_renderer:
4796             if not isinstance(menu_item, dict):
4797                 continue
4798             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4799             text = try_get(
4800                 nav_item_renderer, lambda x: x['text']['simpleText'], str)
4801             if not text or text.lower() != 'show unavailable videos':
4802                 continue
4803             browse_endpoint = try_get(
4804                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4805             browse_id = browse_endpoint.get('browseId')
4806             params = browse_endpoint.get('params')
4807             break
4808
4809         headers = self.generate_api_headers(
4810             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4811             visitor_data=self._extract_visitor_data(data, ytcfg))
4812         query = {
4813             'params': params or 'wgYCCAA=',
4814             'browseId': browse_id or 'VL%s' % item_id
4815         }
4816         return self._extract_response(
4817             item_id=item_id, headers=headers, query=query,
4818             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4819             note='Downloading API JSON with unavailable videos')
4820
4821     @functools.cached_property
4822     def skip_webpage(self):
4823         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4824
4825     def _extract_webpage(self, url, item_id, fatal=True):
4826         webpage, data = None, None
4827         for retry in self.RetryManager(fatal=fatal):
4828             try:
4829                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
4830                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4831             except ExtractorError as e:
4832                 if isinstance(e.cause, network_exceptions):
4833                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4834                         retry.error = e
4835                         continue
4836                 self._error_or_warning(e, fatal=fatal)
4837                 break
4838
4839             try:
4840                 self._extract_and_report_alerts(data)
4841             except ExtractorError as e:
4842                 self._error_or_warning(e, fatal=fatal)
4843                 break
4844
4845             # Sometimes youtube returns a webpage with incomplete ytInitialData
4846             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4847             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
4848                 retry.error = ExtractorError('Incomplete yt initial data received')
4849                 continue
4850
4851         return webpage, data
4852
4853     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4854         """Use if failed to extract ytcfg (and data) from initial webpage"""
4855         if not ytcfg and self.is_authenticated:
4856             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4857             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4858                 raise ExtractorError(
4859                     f'{msg}. If you are not downloading private content, or '
4860                     'your cookies are only for the first account and channel,'
4861                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4862                     expected=True)
4863             self.report_warning(msg, only_once=True)
4864
4865     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4866         data = None
4867         if not self.skip_webpage:
4868             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4869             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4870             # Reject webpage data if redirected to home page without explicitly requesting
4871             selected_tab = self._extract_selected_tab(traverse_obj(
4872                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4873             if (url != 'https://www.youtube.com/feed/recommended'
4874                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4875                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4876                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4877                 if fatal:
4878                     raise ExtractorError(msg, expected=True)
4879                 self.report_warning(msg, only_once=True)
4880         if not data:
4881             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4882             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4883         return data, ytcfg
4884
4885     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4886         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4887         resolve_response = self._extract_response(
4888             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4889             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4890         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4891         for ep_key, ep in endpoints.items():
4892             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4893             if params:
4894                 return self._extract_response(
4895                     item_id=item_id, query=params, ep=ep, headers=headers,
4896                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4897                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4898         err_note = 'Failed to resolve url (does the playlist exist?)'
4899         if fatal:
4900             raise ExtractorError(err_note, expected=True)
4901         self.report_warning(err_note, item_id)
4902
4903     _SEARCH_PARAMS = None
4904
4905     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4906         data = {'query': query}
4907         if params is NO_DEFAULT:
4908             params = self._SEARCH_PARAMS
4909         if params:
4910             data['params'] = params
4911
4912         content_keys = (
4913             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4914             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4915             # ytmusic search
4916             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4917             ('continuationContents', ),
4918         )
4919         display_id = f'query "{query}"'
4920         check_get_keys = tuple({keys[0] for keys in content_keys})
4921         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4922         self._report_playlist_authcheck(ytcfg, fatal=False)
4923
4924         continuation_list = [None]
4925         search = None
4926         for page_num in itertools.count(1):
4927             data.update(continuation_list[0] or {})
4928             headers = self.generate_api_headers(
4929                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4930             search = self._extract_response(
4931                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4932                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4933             slr_contents = traverse_obj(search, *content_keys)
4934             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4935             if not continuation_list[0]:
4936                 break
4937
4938
4939 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4940     IE_DESC = 'YouTube Tabs'
4941     _VALID_URL = r'''(?x:
4942         https?://
4943             (?:\w+\.)?
4944             (?:
4945                 youtube(?:kids)?\.com|
4946                 %(invidious)s
4947             )/
4948             (?:
4949                 (?P<channel_type>channel|c|user|browse)/|
4950                 (?P<not_channel>
4951                     feed/|hashtag/|
4952                     (?:playlist|watch)\?.*?\blist=
4953                 )|
4954                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4955             )
4956             (?P<id>[^/?\#&]+)
4957     )''' % {
4958         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4959         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4960     }
4961     IE_NAME = 'youtube:tab'
4962
4963     _TESTS = [{
4964         'note': 'playlists, multipage',
4965         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4966         'playlist_mincount': 94,
4967         'info_dict': {
4968             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4969             'title': 'Igor Kleiner - Playlists',
4970             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4971             'uploader': 'Igor Kleiner',
4972             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4973             'channel': 'Igor Kleiner',
4974             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4975             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4976             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4977             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4978             'channel_follower_count': int
4979         },
4980     }, {
4981         'note': 'playlists, multipage, different order',
4982         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4983         'playlist_mincount': 94,
4984         'info_dict': {
4985             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4986             'title': 'Igor Kleiner - Playlists',
4987             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4988             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4989             'uploader': 'Igor Kleiner',
4990             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4991             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4992             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4993             'channel': 'Igor Kleiner',
4994             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4995             'channel_follower_count': int
4996         },
4997     }, {
4998         'note': 'playlists, series',
4999         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5000         'playlist_mincount': 5,
5001         'info_dict': {
5002             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5003             'title': '3Blue1Brown - Playlists',
5004             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5005             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5006             'uploader': '3Blue1Brown',
5007             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5008             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5009             'channel': '3Blue1Brown',
5010             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5011             'tags': ['Mathematics'],
5012             'channel_follower_count': int
5013         },
5014     }, {
5015         'note': 'playlists, singlepage',
5016         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5017         'playlist_mincount': 4,
5018         'info_dict': {
5019             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5020             'title': 'ThirstForScience - Playlists',
5021             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5022             'uploader': 'ThirstForScience',
5023             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5024             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5025             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5026             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5027             'tags': 'count:13',
5028             'channel': 'ThirstForScience',
5029             'channel_follower_count': int
5030         }
5031     }, {
5032         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5033         'only_matching': True,
5034     }, {
5035         'note': 'basic, single video playlist',
5036         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5037         'info_dict': {
5038             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5039             'uploader': 'Sergey M.',
5040             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5041             'title': 'youtube-dl public playlist',
5042             'description': '',
5043             'tags': [],
5044             'view_count': int,
5045             'modified_date': '20201130',
5046             'channel': 'Sergey M.',
5047             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5048             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5049             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5050             'availability': 'public',
5051         },
5052         'playlist_count': 1,
5053     }, {
5054         'note': 'empty playlist',
5055         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5056         'info_dict': {
5057             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5058             'uploader': 'Sergey M.',
5059             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5060             'title': 'youtube-dl empty playlist',
5061             'tags': [],
5062             'channel': 'Sergey M.',
5063             'description': '',
5064             'modified_date': '20160902',
5065             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5066             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5067             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5068             'availability': 'public',
5069         },
5070         'playlist_count': 0,
5071     }, {
5072         'note': 'Home tab',
5073         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5074         'info_dict': {
5075             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5076             'title': 'lex will - Home',
5077             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5078             'uploader': 'lex will',
5079             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5080             'channel': 'lex will',
5081             'tags': ['bible', 'history', 'prophesy'],
5082             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5083             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5084             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5085             'channel_follower_count': int
5086         },
5087         'playlist_mincount': 2,
5088     }, {
5089         'note': 'Videos tab',
5090         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5091         'info_dict': {
5092             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5093             'title': 'lex will - Videos',
5094             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5095             'uploader': 'lex will',
5096             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5097             'tags': ['bible', 'history', 'prophesy'],
5098             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5099             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5100             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5101             'channel': 'lex will',
5102             'channel_follower_count': int
5103         },
5104         'playlist_mincount': 975,
5105     }, {
5106         'note': 'Videos tab, sorted by popular',
5107         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5108         'info_dict': {
5109             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5110             'title': 'lex will - Videos',
5111             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5112             'uploader': 'lex will',
5113             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5114             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5115             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5116             'channel': 'lex will',
5117             'tags': ['bible', 'history', 'prophesy'],
5118             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5119             'channel_follower_count': int
5120         },
5121         'playlist_mincount': 199,
5122     }, {
5123         'note': 'Playlists tab',
5124         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5125         'info_dict': {
5126             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5127             'title': 'lex will - Playlists',
5128             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5129             'uploader': 'lex will',
5130             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5131             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5132             'channel': 'lex will',
5133             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5134             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5135             'tags': ['bible', 'history', 'prophesy'],
5136             'channel_follower_count': int
5137         },
5138         'playlist_mincount': 17,
5139     }, {
5140         'note': 'Community tab',
5141         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5142         'info_dict': {
5143             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5144             'title': 'lex will - Community',
5145             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5146             'uploader': 'lex will',
5147             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5148             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5149             'channel': 'lex will',
5150             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5151             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5152             'tags': ['bible', 'history', 'prophesy'],
5153             'channel_follower_count': int
5154         },
5155         'playlist_mincount': 18,
5156     }, {
5157         'note': 'Channels tab',
5158         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5159         'info_dict': {
5160             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5161             'title': 'lex will - Channels',
5162             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5163             'uploader': 'lex will',
5164             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5165             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5166             'channel': 'lex will',
5167             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5168             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5169             'tags': ['bible', 'history', 'prophesy'],
5170             'channel_follower_count': int
5171         },
5172         'playlist_mincount': 12,
5173     }, {
5174         'note': 'Search tab',
5175         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5176         'playlist_mincount': 40,
5177         'info_dict': {
5178             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5179             'title': '3Blue1Brown - Search - linear algebra',
5180             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5181             'uploader': '3Blue1Brown',
5182             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5183             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5184             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5185             'tags': ['Mathematics'],
5186             'channel': '3Blue1Brown',
5187             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5188             'channel_follower_count': int
5189         },
5190     }, {
5191         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5192         'only_matching': True,
5193     }, {
5194         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5195         'only_matching': True,
5196     }, {
5197         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5198         'only_matching': True,
5199     }, {
5200         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5201         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5202         'info_dict': {
5203             'title': '29C3: Not my department',
5204             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5205             'uploader': 'Christiaan008',
5206             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5207             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5208             'tags': [],
5209             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5210             'view_count': int,
5211             'modified_date': '20150605',
5212             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5213             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5214             'channel': 'Christiaan008',
5215             'availability': 'public',
5216         },
5217         'playlist_count': 96,
5218     }, {
5219         'note': 'Large playlist',
5220         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5221         'info_dict': {
5222             'title': 'Uploads from Cauchemar',
5223             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5224             'uploader': 'Cauchemar',
5225             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5226             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5227             'tags': [],
5228             'modified_date': r're:\d{8}',
5229             'channel': 'Cauchemar',
5230             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5231             'view_count': int,
5232             'description': '',
5233             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5234             'availability': 'public',
5235         },
5236         'playlist_mincount': 1123,
5237         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5238     }, {
5239         'note': 'even larger playlist, 8832 videos',
5240         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5241         'only_matching': True,
5242     }, {
5243         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5244         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5245         'info_dict': {
5246             'title': 'Uploads from Interstellar Movie',
5247             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5248             'uploader': 'Interstellar Movie',
5249             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5250             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5251             'tags': [],
5252             'view_count': int,
5253             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5254             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5255             'channel': 'Interstellar Movie',
5256             'description': '',
5257             'modified_date': r're:\d{8}',
5258             'availability': 'public',
5259         },
5260         'playlist_mincount': 21,
5261     }, {
5262         'note': 'Playlist with "show unavailable videos" button',
5263         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5264         'info_dict': {
5265             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5266             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5267             'uploader': 'Phim Siêu Nhân Nhật Bản',
5268             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5269             'view_count': int,
5270             'channel': 'Phim Siêu Nhân Nhật Bản',
5271             'tags': [],
5272             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5273             'description': '',
5274             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5275             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5276             'modified_date': r're:\d{8}',
5277             'availability': 'public',
5278         },
5279         'playlist_mincount': 200,
5280         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5281     }, {
5282         'note': 'Playlist with unavailable videos in page 7',
5283         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5284         'info_dict': {
5285             'title': 'Uploads from BlankTV',
5286             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5287             'uploader': 'BlankTV',
5288             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5289             'channel': 'BlankTV',
5290             'channel_url': 'https://www.youtube.com/c/blanktv',
5291             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5292             'view_count': int,
5293             'tags': [],
5294             'uploader_url': 'https://www.youtube.com/c/blanktv',
5295             'modified_date': r're:\d{8}',
5296             'description': '',
5297             'availability': 'public',
5298         },
5299         'playlist_mincount': 1000,
5300         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5301     }, {
5302         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5303         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5304         'info_dict': {
5305             'title': 'Data Analysis with Dr Mike Pound',
5306             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5307             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5308             'uploader': 'Computerphile',
5309             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5310             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5311             'tags': [],
5312             'view_count': int,
5313             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5314             'channel_url': 'https://www.youtube.com/user/Computerphile',
5315             'channel': 'Computerphile',
5316             'availability': 'public',
5317         },
5318         'playlist_mincount': 11,
5319     }, {
5320         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5321         'only_matching': True,
5322     }, {
5323         'note': 'Playlist URL that does not actually serve a playlist',
5324         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5325         'info_dict': {
5326             'id': 'FqZTN594JQw',
5327             'ext': 'webm',
5328             'title': "Smiley's People 01 detective, Adventure Series, Action",
5329             'uploader': 'STREEM',
5330             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5331             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5332             'upload_date': '20150526',
5333             'license': 'Standard YouTube License',
5334             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5335             'categories': ['People & Blogs'],
5336             'tags': list,
5337             'view_count': int,
5338             'like_count': int,
5339         },
5340         'params': {
5341             'skip_download': True,
5342         },
5343         'skip': 'This video is not available.',
5344         'add_ie': [YoutubeIE.ie_key()],
5345     }, {
5346         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5347         'only_matching': True,
5348     }, {
5349         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5350         'only_matching': True,
5351     }, {
5352         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5353         'info_dict': {
5354             'id': 'Wq15eF5vCbI',  # This will keep changing
5355             'ext': 'mp4',
5356             'title': str,
5357             'uploader': 'Sky News',
5358             'uploader_id': 'skynews',
5359             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5360             'upload_date': r're:\d{8}',
5361             'description': str,
5362             'categories': ['News & Politics'],
5363             'tags': list,
5364             'like_count': int,
5365             'release_timestamp': 1642502819,
5366             'channel': 'Sky News',
5367             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5368             'age_limit': 0,
5369             'view_count': int,
5370             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5371             'playable_in_embed': True,
5372             'release_date': '20220118',
5373             'availability': 'public',
5374             'live_status': 'is_live',
5375             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5376             'channel_follower_count': int
5377         },
5378         'params': {
5379             'skip_download': True,
5380         },
5381         'expected_warnings': ['Ignoring subtitle tracks found in '],
5382     }, {
5383         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5384         'info_dict': {
5385             'id': 'a48o2S1cPoo',
5386             'ext': 'mp4',
5387             'title': 'The Young Turks - Live Main Show',
5388             'uploader': 'The Young Turks',
5389             'uploader_id': 'TheYoungTurks',
5390             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5391             'upload_date': '20150715',
5392             'license': 'Standard YouTube License',
5393             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5394             'categories': ['News & Politics'],
5395             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5396             'like_count': int,
5397         },
5398         'params': {
5399             'skip_download': True,
5400         },
5401         'only_matching': True,
5402     }, {
5403         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5404         'only_matching': True,
5405     }, {
5406         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5407         'only_matching': True,
5408     }, {
5409         'note': 'A channel that is not live. Should raise error',
5410         'url': 'https://www.youtube.com/user/numberphile/live',
5411         'only_matching': True,
5412     }, {
5413         'url': 'https://www.youtube.com/feed/trending',
5414         'only_matching': True,
5415     }, {
5416         'url': 'https://www.youtube.com/feed/library',
5417         'only_matching': True,
5418     }, {
5419         'url': 'https://www.youtube.com/feed/history',
5420         'only_matching': True,
5421     }, {
5422         'url': 'https://www.youtube.com/feed/subscriptions',
5423         'only_matching': True,
5424     }, {
5425         'url': 'https://www.youtube.com/feed/watch_later',
5426         'only_matching': True,
5427     }, {
5428         'note': 'Recommended - redirects to home page.',
5429         'url': 'https://www.youtube.com/feed/recommended',
5430         'only_matching': True,
5431     }, {
5432         'note': 'inline playlist with not always working continuations',
5433         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5434         'only_matching': True,
5435     }, {
5436         'url': 'https://www.youtube.com/course',
5437         'only_matching': True,
5438     }, {
5439         'url': 'https://www.youtube.com/zsecurity',
5440         'only_matching': True,
5441     }, {
5442         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5443         'only_matching': True,
5444     }, {
5445         'url': 'https://www.youtube.com/TheYoungTurks/live',
5446         'only_matching': True,
5447     }, {
5448         'url': 'https://www.youtube.com/hashtag/cctv9',
5449         'info_dict': {
5450             'id': 'cctv9',
5451             'title': '#cctv9',
5452             'tags': [],
5453         },
5454         'playlist_mincount': 350,
5455     }, {
5456         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5457         'only_matching': True,
5458     }, {
5459         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5460         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5461         'only_matching': True
5462     }, {
5463         'note': '/browse/ should redirect to /channel/',
5464         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5465         'only_matching': True
5466     }, {
5467         'note': 'VLPL, should redirect to playlist?list=PL...',
5468         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5469         'info_dict': {
5470             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5471             'uploader': 'NoCopyrightSounds',
5472             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5473             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5474             'title': 'NCS : All Releases 💿',
5475             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5476             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5477             'modified_date': r're:\d{8}',
5478             'view_count': int,
5479             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5480             'tags': [],
5481             'channel': 'NoCopyrightSounds',
5482             'availability': 'public',
5483         },
5484         'playlist_mincount': 166,
5485         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5486     }, {
5487         'note': 'Topic, should redirect to playlist?list=UU...',
5488         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5489         'info_dict': {
5490             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5491             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5492             'title': 'Uploads from Royalty Free Music - Topic',
5493             'uploader': 'Royalty Free Music - Topic',
5494             'tags': [],
5495             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5496             'channel': 'Royalty Free Music - Topic',
5497             'view_count': int,
5498             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5499             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5500             'modified_date': r're:\d{8}',
5501             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5502             'description': '',
5503             'availability': 'public',
5504         },
5505         'expected_warnings': [
5506             'The URL does not have a videos tab',
5507             r'[Uu]navailable videos (are|will be) hidden',
5508         ],
5509         'playlist_mincount': 101,
5510     }, {
5511         'note': 'Topic without a UU playlist',
5512         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5513         'info_dict': {
5514             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5515             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5516             'tags': [],
5517         },
5518         'expected_warnings': [
5519             'the playlist redirect gave error',
5520         ],
5521         'playlist_mincount': 9,
5522     }, {
5523         'note': 'Youtube music Album',
5524         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5525         'info_dict': {
5526             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5527             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5528             'tags': [],
5529             'view_count': int,
5530             'description': '',
5531             'availability': 'unlisted',
5532             'modified_date': r're:\d{8}',
5533         },
5534         'playlist_count': 50,
5535     }, {
5536         'note': 'unlisted single video playlist',
5537         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5538         'info_dict': {
5539             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5540             'uploader': 'colethedj',
5541             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5542             'title': 'yt-dlp unlisted playlist test',
5543             'availability': 'unlisted',
5544             'tags': [],
5545             'modified_date': '20220418',
5546             'channel': 'colethedj',
5547             'view_count': int,
5548             'description': '',
5549             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5550             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5551             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5552         },
5553         'playlist_count': 1,
5554     }, {
5555         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5556         'url': 'https://www.youtube.com/feed/recommended',
5557         'info_dict': {
5558             'id': 'recommended',
5559             'title': 'recommended',
5560             'tags': [],
5561         },
5562         'playlist_mincount': 50,
5563         'params': {
5564             'skip_download': True,
5565             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5566         },
5567     }, {
5568         'note': 'API Fallback: /videos tab, sorted by oldest first',
5569         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5570         'info_dict': {
5571             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5572             'title': 'Cody\'sLab - Videos',
5573             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5574             'uploader': 'Cody\'sLab',
5575             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5576             'channel': 'Cody\'sLab',
5577             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5578             'tags': [],
5579             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5580             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5581             'channel_follower_count': int
5582         },
5583         'playlist_mincount': 650,
5584         'params': {
5585             'skip_download': True,
5586             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5587         },
5588     }, {
5589         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5590         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5591         'info_dict': {
5592             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5593             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5594             'title': 'Uploads from Royalty Free Music - Topic',
5595             'uploader': 'Royalty Free Music - Topic',
5596             'modified_date': r're:\d{8}',
5597             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5598             'description': '',
5599             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5600             'tags': [],
5601             'channel': 'Royalty Free Music - Topic',
5602             'view_count': int,
5603             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5604             'availability': 'public',
5605         },
5606         'expected_warnings': [
5607             'does not have a videos tab',
5608             r'[Uu]navailable videos (are|will be) hidden',
5609         ],
5610         'playlist_mincount': 101,
5611         'params': {
5612             'skip_download': True,
5613             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5614         },
5615     }, {
5616         'note': 'non-standard redirect to regional channel',
5617         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5618         'only_matching': True
5619     }, {
5620         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5621         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5622         'info_dict': {
5623             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5624             'modified_date': '20220407',
5625             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5626             'tags': [],
5627             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5628             'uploader': 'pukkandan',
5629             'availability': 'unlisted',
5630             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5631             'channel': 'pukkandan',
5632             'description': 'Test for collaborative playlist',
5633             'title': 'yt-dlp test - collaborative playlist',
5634             'view_count': int,
5635             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5636         },
5637         'playlist_mincount': 2
5638     }, {
5639         'note': 'translated tab name',
5640         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5641         'info_dict': {
5642             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5643             'tags': [],
5644             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5645             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5646             'description': '',
5647             'title': 'cole-dlp-test-acc - 再生リスト',
5648             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5649             'uploader': 'cole-dlp-test-acc',
5650             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5651             'channel': 'cole-dlp-test-acc',
5652         },
5653         'playlist_mincount': 1,
5654         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5655         'expected_warnings': ['Preferring "ja"'],
5656     }, {
5657         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5658         'note': 'preferred lang set with playlist with translated video titles',
5659         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5660         'info_dict': {
5661             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5662             'tags': [],
5663             'view_count': int,
5664             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5665             'uploader': 'cole-dlp-test-acc',
5666             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5667             'channel': 'cole-dlp-test-acc',
5668             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5669             'description': 'test',
5670             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5671             'title': 'dlp test playlist',
5672             'availability': 'public',
5673         },
5674         'playlist_mincount': 1,
5675         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5676         'expected_warnings': ['Preferring "ja"'],
5677     }, {
5678         # shorts audio pivot for 2GtVksBMYFM.
5679         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5680         'info_dict': {
5681             'id': 'sfv_audio_pivot',
5682             'title': 'sfv_audio_pivot',
5683             'tags': [],
5684         },
5685         'playlist_mincount': 50,
5686
5687     }]
5688
5689     @classmethod
5690     def suitable(cls, url):
5691         return False if YoutubeIE.suitable(url) else super().suitable(url)
5692
5693     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5694
5695     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5696     def _real_extract(self, url, smuggled_data):
5697         item_id = self._match_id(url)
5698         url = urllib.parse.urlunparse(
5699             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5700         compat_opts = self.get_param('compat_opts', [])
5701
5702         def get_mobj(url):
5703             mobj = self._URL_RE.match(url).groupdict()
5704             mobj.update((k, '') for k, v in mobj.items() if v is None)
5705             return mobj
5706
5707         mobj, redirect_warning = get_mobj(url), None
5708         # Youtube returns incomplete data if tabname is not lower case
5709         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5710         if is_channel:
5711             if smuggled_data.get('is_music_url'):
5712                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5713                     item_id = item_id[2:]
5714                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5715                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5716                     mdata = self._extract_tab_endpoint(
5717                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5718                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5719                                         get_all=False, expected_type=str)
5720                     if not murl:
5721                         raise ExtractorError('Failed to resolve album to playlist')
5722                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5723                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5724                     pre = f'https://www.youtube.com/channel/{item_id}'
5725
5726         original_tab_name = tab
5727         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5728             # Home URLs should redirect to /videos/
5729             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5730                                 'To download only the videos in the home page, add a "/featured" to the URL')
5731             tab = '/videos'
5732
5733         url = ''.join((pre, tab, post))
5734         mobj = get_mobj(url)
5735
5736         # Handle both video/playlist URLs
5737         qs = parse_qs(url)
5738         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5739
5740         if not video_id and mobj['not_channel'].startswith('watch'):
5741             if not playlist_id:
5742                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5743                 raise ExtractorError('Unable to recognize tab page')
5744             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5745             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5746             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5747             mobj = get_mobj(url)
5748
5749         if video_id and playlist_id:
5750             if self.get_param('noplaylist'):
5751                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5752                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5753                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5754             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5755
5756         data, ytcfg = self._extract_data(url, item_id)
5757
5758         # YouTube may provide a non-standard redirect to the regional channel
5759         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5760         redirect_url = traverse_obj(
5761             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5762         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5763             redirect_url = ''.join((
5764                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5765             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5766             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5767
5768         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5769         if tabs:
5770             selected_tab = self._extract_selected_tab(tabs)
5771             selected_tab_url = urljoin(
5772                 url, traverse_obj(selected_tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
5773             translated_tab_name = selected_tab.get('title', '').lower()
5774
5775             # Prefer tab name from tab url as it is always in en,
5776             # but only when preferred lang is set as it may not extract reliably in all cases.
5777             selected_tab_name = (self._preferred_lang in (None, 'en') and translated_tab_name
5778                                  or selected_tab_url and get_mobj(selected_tab_url)['tab'][1:]  # primary
5779                                  or translated_tab_name)
5780
5781             if selected_tab_name == 'home':
5782                 selected_tab_name = 'featured'
5783             requested_tab_name = mobj['tab'][1:]
5784
5785             if 'no-youtube-channel-redirect' not in compat_opts:
5786                 if requested_tab_name == 'live':  # Live tab should have redirected to the video
5787                     raise UserNotLive(video_id=mobj['id'])
5788                 if requested_tab_name not in ('', selected_tab_name):
5789                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5790                     if not original_tab_name:
5791                         if item_id[:2] == 'UC':
5792                             # Topic channels don't have /videos. Use the equivalent playlist instead
5793                             pl_id = f'UU{item_id[2:]}'
5794                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5795                             try:
5796                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5797                             except ExtractorError:
5798                                 redirect_warning += ' and the playlist redirect gave error'
5799                             else:
5800                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5801                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5802                         if selected_tab_name and selected_tab_name != requested_tab_name:
5803                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5804                     else:
5805                         raise ExtractorError(redirect_warning, expected=True)
5806
5807         if redirect_warning:
5808             self.to_screen(redirect_warning)
5809         self.write_debug(f'Final URL: {url}')
5810
5811         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5812         if 'no-youtube-unavailable-videos' not in compat_opts:
5813             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5814         self._extract_and_report_alerts(data, only_once=True)
5815         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5816         if tabs:
5817             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5818
5819         playlist = traverse_obj(
5820             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5821         if playlist:
5822             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5823
5824         video_id = traverse_obj(
5825             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5826         if video_id:
5827             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5828                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5829             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5830                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5831
5832         raise ExtractorError('Unable to recognize tab page')
5833
5834
5835 class YoutubePlaylistIE(InfoExtractor):
5836     IE_DESC = 'YouTube playlists'
5837     _VALID_URL = r'''(?x)(?:
5838                         (?:https?://)?
5839                         (?:\w+\.)?
5840                         (?:
5841                             (?:
5842                                 youtube(?:kids)?\.com|
5843                                 %(invidious)s
5844                             )
5845                             /.*?\?.*?\blist=
5846                         )?
5847                         (?P<id>%(playlist_id)s)
5848                      )''' % {
5849         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5850         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5851     }
5852     IE_NAME = 'youtube:playlist'
5853     _TESTS = [{
5854         'note': 'issue #673',
5855         'url': 'PLBB231211A4F62143',
5856         'info_dict': {
5857             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5858             'id': 'PLBB231211A4F62143',
5859             'uploader': 'Wickman',
5860             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5861             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5862             'view_count': int,
5863             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5864             'modified_date': r're:\d{8}',
5865             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5866             'channel': 'Wickman',
5867             'tags': [],
5868             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5869         },
5870         'playlist_mincount': 29,
5871     }, {
5872         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5873         'info_dict': {
5874             'title': 'YDL_safe_search',
5875             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5876         },
5877         'playlist_count': 2,
5878         'skip': 'This playlist is private',
5879     }, {
5880         'note': 'embedded',
5881         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5882         'playlist_count': 4,
5883         'info_dict': {
5884             'title': 'JODA15',
5885             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5886             'uploader': 'milan',
5887             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5888             'description': '',
5889             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5890             'tags': [],
5891             'modified_date': '20140919',
5892             'view_count': int,
5893             'channel': 'milan',
5894             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5895             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5896             'availability': 'public',
5897         },
5898         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5899     }, {
5900         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5901         'playlist_mincount': 455,
5902         'info_dict': {
5903             'title': '2018 Chinese New Singles (11/6 updated)',
5904             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5905             'uploader': 'LBK',
5906             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5907             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5908             'channel': 'LBK',
5909             'view_count': int,
5910             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5911             'tags': [],
5912             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5913             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5914             'modified_date': r're:\d{8}',
5915             'availability': 'public',
5916         },
5917         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5918     }, {
5919         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5920         'only_matching': True,
5921     }, {
5922         # music album playlist
5923         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5924         'only_matching': True,
5925     }]
5926
5927     @classmethod
5928     def suitable(cls, url):
5929         if YoutubeTabIE.suitable(url):
5930             return False
5931         from ..utils import parse_qs
5932         qs = parse_qs(url)
5933         if qs.get('v', [None])[0]:
5934             return False
5935         return super().suitable(url)
5936
5937     def _real_extract(self, url):
5938         playlist_id = self._match_id(url)
5939         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5940         url = update_url_query(
5941             'https://www.youtube.com/playlist',
5942             parse_qs(url) or {'list': playlist_id})
5943         if is_music_url:
5944             url = smuggle_url(url, {'is_music_url': True})
5945         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5946
5947
5948 class YoutubeYtBeIE(InfoExtractor):
5949     IE_DESC = 'youtu.be'
5950     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5951     _TESTS = [{
5952         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5953         'info_dict': {
5954             'id': 'yeWKywCrFtk',
5955             'ext': 'mp4',
5956             'title': 'Small Scale Baler and Braiding Rugs',
5957             'uploader': 'Backus-Page House Museum',
5958             'uploader_id': 'backuspagemuseum',
5959             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5960             'upload_date': '20161008',
5961             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5962             'categories': ['Nonprofits & Activism'],
5963             'tags': list,
5964             'like_count': int,
5965             'age_limit': 0,
5966             'playable_in_embed': True,
5967             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5968             'channel': 'Backus-Page House Museum',
5969             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5970             'live_status': 'not_live',
5971             'view_count': int,
5972             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5973             'availability': 'public',
5974             'duration': 59,
5975             'comment_count': int,
5976             'channel_follower_count': int
5977         },
5978         'params': {
5979             'noplaylist': True,
5980             'skip_download': True,
5981         },
5982     }, {
5983         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5984         'only_matching': True,
5985     }]
5986
5987     def _real_extract(self, url):
5988         mobj = self._match_valid_url(url)
5989         video_id = mobj.group('id')
5990         playlist_id = mobj.group('playlist_id')
5991         return self.url_result(
5992             update_url_query('https://www.youtube.com/watch', {
5993                 'v': video_id,
5994                 'list': playlist_id,
5995                 'feature': 'youtu.be',
5996             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5997
5998
5999 class YoutubeLivestreamEmbedIE(InfoExtractor):
6000     IE_DESC = 'YouTube livestream embeds'
6001     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6002     _TESTS = [{
6003         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6004         'only_matching': True,
6005     }]
6006
6007     def _real_extract(self, url):
6008         channel_id = self._match_id(url)
6009         return self.url_result(
6010             f'https://www.youtube.com/channel/{channel_id}/live',
6011             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6012
6013
6014 class YoutubeYtUserIE(InfoExtractor):
6015     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6016     IE_NAME = 'youtube:user'
6017     _VALID_URL = r'ytuser:(?P<id>.+)'
6018     _TESTS = [{
6019         'url': 'ytuser:phihag',
6020         'only_matching': True,
6021     }]
6022
6023     def _real_extract(self, url):
6024         user_id = self._match_id(url)
6025         return self.url_result(
6026             'https://www.youtube.com/user/%s/videos' % user_id,
6027             ie=YoutubeTabIE.ie_key(), video_id=user_id)
6028
6029
6030 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6031     IE_NAME = 'youtube:favorites'
6032     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6033     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6034     _LOGIN_REQUIRED = True
6035     _TESTS = [{
6036         'url': ':ytfav',
6037         'only_matching': True,
6038     }, {
6039         'url': ':ytfavorites',
6040         'only_matching': True,
6041     }]
6042
6043     def _real_extract(self, url):
6044         return self.url_result(
6045             'https://www.youtube.com/playlist?list=LL',
6046             ie=YoutubeTabIE.ie_key())
6047
6048
6049 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6050     IE_NAME = 'youtube:notif'
6051     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6052     _VALID_URL = r':ytnotif(?:ication)?s?'
6053     _LOGIN_REQUIRED = True
6054     _TESTS = [{
6055         'url': ':ytnotif',
6056         'only_matching': True,
6057     }, {
6058         'url': ':ytnotifications',
6059         'only_matching': True,
6060     }]
6061
6062     def _extract_notification_menu(self, response, continuation_list):
6063         notification_list = traverse_obj(
6064             response,
6065             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6066             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6067             expected_type=list) or []
6068         continuation_list[0] = None
6069         for item in notification_list:
6070             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6071             if entry:
6072                 yield entry
6073             continuation = item.get('continuationItemRenderer')
6074             if continuation:
6075                 continuation_list[0] = continuation
6076
6077     def _extract_notification_renderer(self, notification):
6078         video_id = traverse_obj(
6079             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6080         url = f'https://www.youtube.com/watch?v={video_id}'
6081         channel_id = None
6082         if not video_id:
6083             browse_ep = traverse_obj(
6084                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6085             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6086             post_id = self._search_regex(
6087                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6088                 'post id', default=None)
6089             if not channel_id or not post_id:
6090                 return
6091             # The direct /post url redirects to this in the browser
6092             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6093
6094         channel = traverse_obj(
6095             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6096             expected_type=str)
6097         notification_title = self._get_text(notification, 'shortMessage')
6098         if notification_title:
6099             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6100         # TODO: handle recommended videos
6101         title = self._search_regex(
6102             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6103             'video title', default=None)
6104         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6105                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6106                      else None)
6107         return {
6108             '_type': 'url',
6109             'url': url,
6110             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6111             'video_id': video_id,
6112             'title': title,
6113             'channel_id': channel_id,
6114             'channel': channel,
6115             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6116             'timestamp': timestamp,
6117         }
6118
6119     def _notification_menu_entries(self, ytcfg):
6120         continuation_list = [None]
6121         response = None
6122         for page in itertools.count(1):
6123             ctoken = traverse_obj(
6124                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6125             response = self._extract_response(
6126                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6127                 ep='notification/get_notification_menu', check_get_keys='actions',
6128                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6129             yield from self._extract_notification_menu(response, continuation_list)
6130             if not continuation_list[0]:
6131                 break
6132
6133     def _real_extract(self, url):
6134         display_id = 'notifications'
6135         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6136         self._report_playlist_authcheck(ytcfg)
6137         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6138
6139
6140 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6141     IE_DESC = 'YouTube search'
6142     IE_NAME = 'youtube:search'
6143     _SEARCH_KEY = 'ytsearch'
6144     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6145     _TESTS = [{
6146         'url': 'ytsearch5:youtube-dl test video',
6147         'playlist_count': 5,
6148         'info_dict': {
6149             'id': 'youtube-dl test video',
6150             'title': 'youtube-dl test video',
6151         }
6152     }]
6153
6154
6155 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6156     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6157     _SEARCH_KEY = 'ytsearchdate'
6158     IE_DESC = 'YouTube search, newest videos first'
6159     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6160     _TESTS = [{
6161         'url': 'ytsearchdate5:youtube-dl test video',
6162         'playlist_count': 5,
6163         'info_dict': {
6164             'id': 'youtube-dl test video',
6165             'title': 'youtube-dl test video',
6166         }
6167     }]
6168
6169
6170 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6171     IE_DESC = 'YouTube search URLs with sorting and filter support'
6172     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6173     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6174     _TESTS = [{
6175         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6176         'playlist_mincount': 5,
6177         'info_dict': {
6178             'id': 'youtube-dl test video',
6179             'title': 'youtube-dl test video',
6180         }
6181     }, {
6182         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6183         'playlist_mincount': 5,
6184         'info_dict': {
6185             'id': 'python',
6186             'title': 'python',
6187         }
6188     }, {
6189         'url': 'https://www.youtube.com/results?search_query=%23cats',
6190         'playlist_mincount': 1,
6191         'info_dict': {
6192             'id': '#cats',
6193             'title': '#cats',
6194             # The test suite does not have support for nested playlists
6195             # 'entries': [{
6196             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6197             #     'title': '#cats',
6198             # }],
6199         },
6200     }, {
6201         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6202         'only_matching': True,
6203     }]
6204
6205     def _real_extract(self, url):
6206         qs = parse_qs(url)
6207         query = (qs.get('search_query') or qs.get('q'))[0]
6208         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6209
6210
6211 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6212     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6213     IE_NAME = 'youtube:music:search_url'
6214     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6215     _TESTS = [{
6216         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6217         'playlist_count': 16,
6218         'info_dict': {
6219             'id': 'royalty free music',
6220             'title': 'royalty free music',
6221         }
6222     }, {
6223         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6224         'playlist_mincount': 30,
6225         'info_dict': {
6226             'id': 'royalty free music - songs',
6227             'title': 'royalty free music - songs',
6228         },
6229         'params': {'extract_flat': 'in_playlist'}
6230     }, {
6231         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6232         'playlist_mincount': 30,
6233         'info_dict': {
6234             'id': 'royalty free music - community playlists',
6235             'title': 'royalty free music - community playlists',
6236         },
6237         'params': {'extract_flat': 'in_playlist'}
6238     }]
6239
6240     _SECTIONS = {
6241         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6242         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6243         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6244         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6245         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6246         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6247     }
6248
6249     def _real_extract(self, url):
6250         qs = parse_qs(url)
6251         query = (qs.get('search_query') or qs.get('q'))[0]
6252         params = qs.get('sp', (None,))[0]
6253         if params:
6254             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6255         else:
6256             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6257             params = self._SECTIONS.get(section)
6258             if not params:
6259                 section = None
6260         title = join_nonempty(query, section, delim=' - ')
6261         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6262
6263
6264 class YoutubeFeedsInfoExtractor(InfoExtractor):
6265     """
6266     Base class for feed extractors
6267     Subclasses must re-define the _FEED_NAME property.
6268     """
6269     _LOGIN_REQUIRED = True
6270     _FEED_NAME = 'feeds'
6271
6272     def _real_initialize(self):
6273         YoutubeBaseInfoExtractor._check_login_required(self)
6274
6275     @classproperty
6276     def IE_NAME(self):
6277         return f'youtube:{self._FEED_NAME}'
6278
6279     def _real_extract(self, url):
6280         return self.url_result(
6281             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6282
6283
6284 class YoutubeWatchLaterIE(InfoExtractor):
6285     IE_NAME = 'youtube:watchlater'
6286     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6287     _VALID_URL = r':ytwatchlater'
6288     _TESTS = [{
6289         'url': ':ytwatchlater',
6290         'only_matching': True,
6291     }]
6292
6293     def _real_extract(self, url):
6294         return self.url_result(
6295             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6296
6297
6298 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6299     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6300     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6301     _FEED_NAME = 'recommended'
6302     _LOGIN_REQUIRED = False
6303     _TESTS = [{
6304         'url': ':ytrec',
6305         'only_matching': True,
6306     }, {
6307         'url': ':ytrecommended',
6308         'only_matching': True,
6309     }, {
6310         'url': 'https://youtube.com',
6311         'only_matching': True,
6312     }]
6313
6314
6315 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6316     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6317     _VALID_URL = r':ytsub(?:scription)?s?'
6318     _FEED_NAME = 'subscriptions'
6319     _TESTS = [{
6320         'url': ':ytsubs',
6321         'only_matching': True,
6322     }, {
6323         'url': ':ytsubscriptions',
6324         'only_matching': True,
6325     }]
6326
6327
6328 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6329     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6330     _VALID_URL = r':ythis(?:tory)?'
6331     _FEED_NAME = 'history'
6332     _TESTS = [{
6333         'url': ':ythistory',
6334         'only_matching': True,
6335     }]
6336
6337
6338 class YoutubeStoriesIE(InfoExtractor):
6339     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6340     IE_NAME = 'youtube:stories'
6341     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6342     _TESTS = [{
6343         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6344         'only_matching': True,
6345     }]
6346
6347     def _real_extract(self, url):
6348         playlist_id = f'RLTD{self._match_id(url)}'
6349         return self.url_result(
6350             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6351             ie=YoutubeTabIE, video_id=playlist_id)
6352
6353
6354 class YoutubeShortsAudioPivotIE(InfoExtractor):
6355     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6356     IE_NAME = 'youtube:shorts:pivot:audio'
6357     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6358     _TESTS = [{
6359         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6360         'only_matching': True,
6361     }]
6362
6363     @staticmethod
6364     def _generate_audio_pivot_params(video_id):
6365         """
6366         Generates sfv_audio_pivot browse params for this video id
6367         """
6368         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6369         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6370
6371     def _real_extract(self, url):
6372         video_id = self._match_id(url)
6373         return self.url_result(
6374             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6375             ie=YoutubeTabIE)
6376
6377
6378 class YoutubeTruncatedURLIE(InfoExtractor):
6379     IE_NAME = 'youtube:truncated_url'
6380     IE_DESC = False  # Do not list
6381     _VALID_URL = r'''(?x)
6382         (?:https?://)?
6383         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6384         (?:watch\?(?:
6385             feature=[a-z_]+|
6386             annotation_id=annotation_[^&]+|
6387             x-yt-cl=[0-9]+|
6388             hl=[^&]*|
6389             t=[0-9]+
6390         )?
6391         |
6392             attribution_link\?a=[^&]+
6393         )
6394         $
6395     '''
6396
6397     _TESTS = [{
6398         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6399         'only_matching': True,
6400     }, {
6401         'url': 'https://www.youtube.com/watch?',
6402         'only_matching': True,
6403     }, {
6404         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6405         'only_matching': True,
6406     }, {
6407         'url': 'https://www.youtube.com/watch?feature=foo',
6408         'only_matching': True,
6409     }, {
6410         'url': 'https://www.youtube.com/watch?hl=en-GB',
6411         'only_matching': True,
6412     }, {
6413         'url': 'https://www.youtube.com/watch?t=2372',
6414         'only_matching': True,
6415     }]
6416
6417     def _real_extract(self, url):
6418         raise ExtractorError(
6419             'Did you forget to quote the URL? Remember that & is a meta '
6420             'character in most shells, so you want to put the URL in quotes, '
6421             'like  youtube-dl '
6422             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6423             ' or simply  youtube-dl BaW_jenozKc  .',
6424             expected=True)
6425
6426
6427 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6428     IE_NAME = 'youtube:clip'
6429     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6430     _TESTS = [{
6431         # FIXME: Other metadata should be extracted from the clip, not from the base video
6432         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6433         'info_dict': {
6434             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6435             'ext': 'mp4',
6436             'section_start': 29.0,
6437             'section_end': 39.7,
6438             'duration': 10.7,
6439             'age_limit': 0,
6440             'availability': 'public',
6441             'categories': ['Gaming'],
6442             'channel': 'Scott The Woz',
6443             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6444             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6445             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6446             'like_count': int,
6447             'playable_in_embed': True,
6448             'tags': 'count:17',
6449             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6450             'title': 'Mobile Games on Console - Scott The Woz',
6451             'upload_date': '20210920',
6452             'uploader': 'Scott The Woz',
6453             'uploader_id': 'scottthewoz',
6454             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6455             'view_count': int,
6456             'live_status': 'not_live',
6457             'channel_follower_count': int
6458         }
6459     }]
6460
6461     def _real_extract(self, url):
6462         clip_id = self._match_id(url)
6463         _, data = self._extract_webpage(url, clip_id)
6464
6465         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6466         if not video_id:
6467             raise ExtractorError('Unable to find video ID')
6468
6469         clip_data = traverse_obj(data, (
6470             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6471             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6472             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6473             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6474
6475         return {
6476             '_type': 'url_transparent',
6477             'url': f'https://www.youtube.com/watch?v={video_id}',
6478             'ie_key': YoutubeIE.ie_key(),
6479             'id': clip_id,
6480             'section_start': int(clip_data['startTimeMs']) / 1000,
6481             'section_end': int(clip_data['endTimeMs']) / 1000,
6482         }
6483
6484
6485 class YoutubeTruncatedIDIE(InfoExtractor):
6486     IE_NAME = 'youtube:truncated_id'
6487     IE_DESC = False  # Do not list
6488     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6489
6490     _TESTS = [{
6491         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6492         'only_matching': True,
6493     }]
6494
6495     def _real_extract(self, url):
6496         video_id = self._match_id(url)
6497         raise ExtractorError(
6498             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6499             expected=True)