yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import enum
   6 import hashlib
   7 import itertools
   8 import json
   9 import math
  10 import os.path
  11 import random
  12 import re
  13 import sys
  14 import threading
  15 import time
  16 import traceback
  17 import urllib.error
  18 import urllib.parse
  19
  20 from .common import InfoExtractor, SearchInfoExtractor
  21 from .openload import PhantomJSwrapper
  22 from ..compat import functools
  23 from ..jsinterp import JSInterpreter
  24 from ..utils import (
  25     NO_DEFAULT,
  26     ExtractorError,
  27     UserNotLive,
  28     bug_reports_message,
  29     classproperty,
  30     clean_html,
  31     datetime_from_str,
  32     dict_get,
  33     filter_dict,
  34     float_or_none,
  35     format_field,
  36     get_first,
  37     int_or_none,
  38     is_html,
  39     join_nonempty,
  40     js_to_json,
  41     mimetype2ext,
  42     network_exceptions,
  43     orderedSet,
  44     parse_codecs,
  45     parse_count,
  46     parse_duration,
  47     parse_iso8601,
  48     parse_qs,
  49     qualities,
  50     remove_start,
  51     smuggle_url,
  52     str_or_none,
  53     str_to_int,
  54     strftime_or_none,
  55     traverse_obj,
  56     try_get,
  57     unescapeHTML,
  58     unified_strdate,
  59     unified_timestamp,
  60     unsmuggle_url,
  61     update_url_query,
  62     url_or_none,
  63     urljoin,
  64     variadic,
  65 )
  66
  67 # any clients starting with _ cannot be explicitly requested by the user
  68 INNERTUBE_CLIENTS = {
  69     'web': {
  70         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  71         'INNERTUBE_CONTEXT': {
  72             'client': {
  73                 'clientName': 'WEB',
  74                 'clientVersion': '2.20220801.00.00',
  75             }
  76         },
  77         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  78     },
  79     'web_embedded': {
  80         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  81         'INNERTUBE_CONTEXT': {
  82             'client': {
  83                 'clientName': 'WEB_EMBEDDED_PLAYER',
  84                 'clientVersion': '1.20220731.00.00',
  85             },
  86         },
  87         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  88     },
  89     'web_music': {
  90         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  91         'INNERTUBE_HOST': 'music.youtube.com',
  92         'INNERTUBE_CONTEXT': {
  93             'client': {
  94                 'clientName': 'WEB_REMIX',
  95                 'clientVersion': '1.20220727.01.00',
  96             }
  97         },
  98         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
  99     },
 100     'web_creator': {
 101         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 102         'INNERTUBE_CONTEXT': {
 103             'client': {
 104                 'clientName': 'WEB_CREATOR',
 105                 'clientVersion': '1.20220726.00.00',
 106             }
 107         },
 108         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 109     },
 110     'android': {
 111         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 112         'INNERTUBE_CONTEXT': {
 113             'client': {
 114                 'clientName': 'ANDROID',
 115                 'clientVersion': '17.31.35',
 116                 'androidSdkVersion': 30,
 117                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 118             }
 119         },
 120         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 121         'REQUIRE_JS_PLAYER': False
 122     },
 123     'android_embedded': {
 124         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 125         'INNERTUBE_CONTEXT': {
 126             'client': {
 127                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 128                 'clientVersion': '17.31.35',
 129                 'androidSdkVersion': 30,
 130                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 131             },
 132         },
 133         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 134         'REQUIRE_JS_PLAYER': False
 135     },
 136     'android_music': {
 137         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 138         'INNERTUBE_CONTEXT': {
 139             'client': {
 140                 'clientName': 'ANDROID_MUSIC',
 141                 'clientVersion': '5.16.51',
 142                 'androidSdkVersion': 30,
 143                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 144             }
 145         },
 146         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 147         'REQUIRE_JS_PLAYER': False
 148     },
 149     'android_creator': {
 150         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 151         'INNERTUBE_CONTEXT': {
 152             'client': {
 153                 'clientName': 'ANDROID_CREATOR',
 154                 'clientVersion': '22.30.100',
 155                 'androidSdkVersion': 30,
 156                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 157             },
 158         },
 159         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 160         'REQUIRE_JS_PLAYER': False
 161     },
 162     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 163     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 164     'ios': {
 165         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 166         'INNERTUBE_CONTEXT': {
 167             'client': {
 168                 'clientName': 'IOS',
 169                 'clientVersion': '17.33.2',
 170                 'deviceModel': 'iPhone14,3',
 171                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 172             }
 173         },
 174         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 175         'REQUIRE_JS_PLAYER': False
 176     },
 177     'ios_embedded': {
 178         'INNERTUBE_CONTEXT': {
 179             'client': {
 180                 'clientName': 'IOS_MESSAGES_EXTENSION',
 181                 'clientVersion': '17.33.2',
 182                 'deviceModel': 'iPhone14,3',
 183                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 184             },
 185         },
 186         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 187         'REQUIRE_JS_PLAYER': False
 188     },
 189     'ios_music': {
 190         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 191         'INNERTUBE_CONTEXT': {
 192             'client': {
 193                 'clientName': 'IOS_MUSIC',
 194                 'clientVersion': '5.21',
 195                 'deviceModel': 'iPhone14,3',
 196                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 197             },
 198         },
 199         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 200         'REQUIRE_JS_PLAYER': False
 201     },
 202     'ios_creator': {
 203         'INNERTUBE_CONTEXT': {
 204             'client': {
 205                 'clientName': 'IOS_CREATOR',
 206                 'clientVersion': '22.33.101',
 207                 'deviceModel': 'iPhone14,3',
 208                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 209             },
 210         },
 211         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 212         'REQUIRE_JS_PLAYER': False
 213     },
 214     # mweb has 'ultralow' formats
 215     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 216     'mweb': {
 217         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 218         'INNERTUBE_CONTEXT': {
 219             'client': {
 220                 'clientName': 'MWEB',
 221                 'clientVersion': '2.20220801.00.00',
 222             }
 223         },
 224         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 225     },
 226     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 227     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 228     'tv_embedded': {
 229         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 230         'INNERTUBE_CONTEXT': {
 231             'client': {
 232                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 233                 'clientVersion': '2.0',
 234             },
 235         },
 236         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 237     },
 238 }
 239
 240
 241 def _split_innertube_client(client_name):
 242     variant, *base = client_name.rsplit('.', 1)
 243     if base:
 244         return variant, base[0], variant
 245     base, *variant = client_name.split('_', 1)
 246     return client_name, base, variant[0] if variant else None
 247
 248
 249 def build_innertube_clients():
 250     THIRD_PARTY = {
 251         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 252     }
 253     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 254     priority = qualities(BASE_CLIENTS[::-1])
 255
 256     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 257         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 258         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 259         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 260         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 261
 262         _, base_client, variant = _split_innertube_client(client)
 263         ytcfg['priority'] = 10 * priority(base_client)
 264
 265         if not variant:
 266             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 267             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 268             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 269             embedscreen['priority'] -= 3
 270         elif variant == 'embedded':
 271             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 272             ytcfg['priority'] -= 2
 273         else:
 274             ytcfg['priority'] -= 3
 275
 276
 277 build_innertube_clients()
 278
 279
 280 class BadgeType(enum.Enum):
 281     AVAILABILITY_UNLISTED = enum.auto()
 282     AVAILABILITY_PRIVATE = enum.auto()
 283     AVAILABILITY_PUBLIC = enum.auto()
 284     AVAILABILITY_PREMIUM = enum.auto()
 285     AVAILABILITY_SUBSCRIPTION = enum.auto()
 286     LIVE_NOW = enum.auto()
 287
 288
 289 class YoutubeBaseInfoExtractor(InfoExtractor):
 290     """Provide base functions for Youtube extractors"""
 291
 292     _RESERVED_NAMES = (
 293         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 294         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 295         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 296         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 297
 298     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 299
 300     # _NETRC_MACHINE = 'youtube'
 301
 302     # If True it will raise an error if no login info is provided
 303     _LOGIN_REQUIRED = False
 304
 305     _INVIDIOUS_SITES = (
 306         # invidious-redirect websites
 307         r'(?:www\.)?redirect\.invidious\.io',
 308         r'(?:(?:www|dev)\.)?invidio\.us',
 309         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 310         r'(?:www\.)?invidious\.pussthecat\.org',
 311         r'(?:www\.)?invidious\.zee\.li',
 312         r'(?:www\.)?invidious\.ethibox\.fr',
 313         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 314         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 315         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 316         # youtube-dl invidious instances list
 317         r'(?:(?:www|no)\.)?invidiou\.sh',
 318         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 319         r'(?:www\.)?invidious\.kabi\.tk',
 320         r'(?:www\.)?invidious\.mastodon\.host',
 321         r'(?:www\.)?invidious\.zapashcanon\.fr',
 322         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 323         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 324         r'(?:www\.)?invidious\.himiko\.cloud',
 325         r'(?:www\.)?invidious\.reallyancient\.tech',
 326         r'(?:www\.)?invidious\.tube',
 327         r'(?:www\.)?invidiou\.site',
 328         r'(?:www\.)?invidious\.site',
 329         r'(?:www\.)?invidious\.xyz',
 330         r'(?:www\.)?invidious\.nixnet\.xyz',
 331         r'(?:www\.)?invidious\.048596\.xyz',
 332         r'(?:www\.)?invidious\.drycat\.fr',
 333         r'(?:www\.)?inv\.skyn3t\.in',
 334         r'(?:www\.)?tube\.poal\.co',
 335         r'(?:www\.)?tube\.connect\.cafe',
 336         r'(?:www\.)?vid\.wxzm\.sx',
 337         r'(?:www\.)?vid\.mint\.lgbt',
 338         r'(?:www\.)?vid\.puffyan\.us',
 339         r'(?:www\.)?yewtu\.be',
 340         r'(?:www\.)?yt\.elukerio\.org',
 341         r'(?:www\.)?yt\.lelux\.fi',
 342         r'(?:www\.)?invidious\.ggc-project\.de',
 343         r'(?:www\.)?yt\.maisputain\.ovh',
 344         r'(?:www\.)?ytprivate\.com',
 345         r'(?:www\.)?invidious\.13ad\.de',
 346         r'(?:www\.)?invidious\.toot\.koeln',
 347         r'(?:www\.)?invidious\.fdn\.fr',
 348         r'(?:www\.)?watch\.nettohikari\.com',
 349         r'(?:www\.)?invidious\.namazso\.eu',
 350         r'(?:www\.)?invidious\.silkky\.cloud',
 351         r'(?:www\.)?invidious\.exonip\.de',
 352         r'(?:www\.)?invidious\.riverside\.rocks',
 353         r'(?:www\.)?invidious\.blamefran\.net',
 354         r'(?:www\.)?invidious\.moomoo\.de',
 355         r'(?:www\.)?ytb\.trom\.tf',
 356         r'(?:www\.)?yt\.cyberhost\.uk',
 357         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 358         r'(?:www\.)?qklhadlycap4cnod\.onion',
 359         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 360         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 361         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 362         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 363         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 364         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 365         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 366         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 367         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 368         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 369         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 370         r'(?:www\.)?piped\.kavin\.rocks',
 371         r'(?:www\.)?piped\.silkky\.cloud',
 372         r'(?:www\.)?piped\.tokhmi\.xyz',
 373         r'(?:www\.)?piped\.moomoo\.me',
 374         r'(?:www\.)?il\.ax',
 375         r'(?:www\.)?piped\.syncpundit\.com',
 376         r'(?:www\.)?piped\.mha\.fi',
 377         r'(?:www\.)?piped\.mint\.lgbt',
 378         r'(?:www\.)?piped\.privacy\.com\.de',
 379     )
 380
 381     # extracted from account/account_menu ep
 382     # XXX: These are the supported YouTube UI and API languages,
 383     # which is slightly different from languages supported for translation in YouTube studio
 384     _SUPPORTED_LANG_CODES = [
 385         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 386         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 387         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 388         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 389         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 390         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 391     ]
 392
 393     @functools.cached_property
 394     def _preferred_lang(self):
 395         """
 396         Returns a language code supported by YouTube for the user preferred language.
 397         Returns None if no preferred language set.
 398         """
 399         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 400         if not preferred_lang:
 401             return
 402         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 403             raise ExtractorError(
 404                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 405                 expected=True)
 406         elif preferred_lang != 'en':
 407             self.report_warning(
 408                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 409         return preferred_lang
 410
 411     def _initialize_consent(self):
 412         cookies = self._get_cookies('https://www.youtube.com/')
 413         if cookies.get('__Secure-3PSID'):
 414             return
 415         consent_id = None
 416         consent = cookies.get('CONSENT')
 417         if consent:
 418             if 'YES' in consent.value:
 419                 return
 420             consent_id = self._search_regex(
 421                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 422         if not consent_id:
 423             consent_id = random.randint(100, 999)
 424         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 425
 426     def _initialize_pref(self):
 427         cookies = self._get_cookies('https://www.youtube.com/')
 428         pref_cookie = cookies.get('PREF')
 429         pref = {}
 430         if pref_cookie:
 431             try:
 432                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 433             except ValueError:
 434                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 435         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 436         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 437
 438     def _real_initialize(self):
 439         self._initialize_pref()
 440         self._initialize_consent()
 441         self._check_login_required()
 442
 443     def _check_login_required(self):
 444         if self._LOGIN_REQUIRED and not self._cookies_passed:
 445             self.raise_login_required('Login details are needed to download this content', method='cookies')
 446
 447     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 448     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 449
 450     def _get_default_ytcfg(self, client='web'):
 451         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 452
 453     def _get_innertube_host(self, client='web'):
 454         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 455
 456     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 457         # try_get but with fallback to default ytcfg client values when present
 458         _func = lambda y: try_get(y, getter, expected_type)
 459         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 460
 461     def _extract_client_name(self, ytcfg, default_client='web'):
 462         return self._ytcfg_get_safe(
 463             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 464                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 465
 466     def _extract_client_version(self, ytcfg, default_client='web'):
 467         return self._ytcfg_get_safe(
 468             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 469                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 470
 471     def _select_api_hostname(self, req_api_hostname, default_client=None):
 472         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 473                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 474
 475     def _extract_api_key(self, ytcfg=None, default_client='web'):
 476         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 477
 478     def _extract_context(self, ytcfg=None, default_client='web'):
 479         context = get_first(
 480             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 481         # Enforce language and tz for extraction
 482         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 483         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 484         return context
 485
 486     _SAPISID = None
 487
 488     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 489         time_now = round(time.time())
 490         if self._SAPISID is None:
 491             yt_cookies = self._get_cookies('https://www.youtube.com')
 492             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 493             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 494             sapisid_cookie = dict_get(
 495                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 496             if sapisid_cookie and sapisid_cookie.value:
 497                 self._SAPISID = sapisid_cookie.value
 498                 self.write_debug('Extracted SAPISID cookie')
 499                 # SAPISID cookie is required if not already present
 500                 if not yt_cookies.get('SAPISID'):
 501                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 502                     self._set_cookie(
 503                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 504             else:
 505                 self._SAPISID = False
 506         if not self._SAPISID:
 507             return None
 508         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 509         sapisidhash = hashlib.sha1(
 510             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 511         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 512
 513     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 514                   note='Downloading API JSON', errnote='Unable to download API page',
 515                   context=None, api_key=None, api_hostname=None, default_client='web'):
 516
 517         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 518         data.update(query)
 519         real_headers = self.generate_api_headers(default_client=default_client)
 520         real_headers.update({'content-type': 'application/json'})
 521         if headers:
 522             real_headers.update(headers)
 523         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 524                    or api_key or self._extract_api_key(default_client=default_client))
 525         return self._download_json(
 526             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 527             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 528             data=json.dumps(data).encode('utf8'), headers=real_headers,
 529             query={'key': api_key, 'prettyPrint': 'false'})
 530
 531     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 532         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 533
 534     @staticmethod
 535     def _extract_session_index(*data):
 536         """
 537         Index of current account in account list.
 538         See: https://github.com/yt-dlp/yt-dlp/pull/519
 539         """
 540         for ytcfg in data:
 541             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 542             if session_index is not None:
 543                 return session_index
 544
 545     # Deprecated?
 546     def _extract_identity_token(self, ytcfg=None, webpage=None):
 547         if ytcfg:
 548             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 549             if token:
 550                 return token
 551         if webpage:
 552             return self._search_regex(
 553                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 554                 'identity token', default=None, fatal=False)
 555
 556     @staticmethod
 557     def _extract_account_syncid(*args):
 558         """
 559         Extract syncId required to download private playlists of secondary channels
 560         @params response and/or ytcfg
 561         """
 562         for data in args:
 563             # ytcfg includes channel_syncid if on secondary channel
 564             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 565             if delegated_sid:
 566                 return delegated_sid
 567             sync_ids = (try_get(
 568                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 569                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 570             if len(sync_ids) >= 2 and sync_ids[1]:
 571                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 572                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 573                 return sync_ids[0]
 574
 575     @staticmethod
 576     def _extract_visitor_data(*args):
 577         """
 578         Extracts visitorData from an API response or ytcfg
 579         Appears to be used to track session state
 580         """
 581         return get_first(
 582             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 583             expected_type=str)
 584
 585     @functools.cached_property
 586     def is_authenticated(self):
 587         return bool(self._generate_sapisidhash_header())
 588
 589     def extract_ytcfg(self, video_id, webpage):
 590         if not webpage:
 591             return {}
 592         return self._parse_json(
 593             self._search_regex(
 594                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 595                 default='{}'), video_id, fatal=False) or {}
 596
 597     def generate_api_headers(
 598             self, *, ytcfg=None, account_syncid=None, session_index=None,
 599             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 600
 601         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 602         headers = {
 603             'X-YouTube-Client-Name': str(
 604                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 605             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 606             'Origin': origin,
 607             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 608             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 609             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 610             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 611         }
 612         if session_index is None:
 613             session_index = self._extract_session_index(ytcfg)
 614         if account_syncid or session_index is not None:
 615             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 616
 617         auth = self._generate_sapisidhash_header(origin)
 618         if auth is not None:
 619             headers['Authorization'] = auth
 620             headers['X-Origin'] = origin
 621         return filter_dict(headers)
 622
 623     def _download_ytcfg(self, client, video_id):
 624         url = {
 625             'web': 'https://www.youtube.com',
 626             'web_music': 'https://music.youtube.com',
 627             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 628         }.get(client)
 629         if not url:
 630             return {}
 631         webpage = self._download_webpage(
 632             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 633         return self.extract_ytcfg(video_id, webpage) or {}
 634
 635     @staticmethod
 636     def _build_api_continuation_query(continuation, ctp=None):
 637         query = {
 638             'continuation': continuation
 639         }
 640         # TODO: Inconsistency with clickTrackingParams.
 641         # Currently we have a fixed ctp contained within context (from ytcfg)
 642         # and a ctp in root query for continuation.
 643         if ctp:
 644             query['clickTracking'] = {'clickTrackingParams': ctp}
 645         return query
 646
 647     @classmethod
 648     def _extract_next_continuation_data(cls, renderer):
 649         next_continuation = try_get(
 650             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 651                        lambda x: x['continuation']['reloadContinuationData']), dict)
 652         if not next_continuation:
 653             return
 654         continuation = next_continuation.get('continuation')
 655         if not continuation:
 656             return
 657         ctp = next_continuation.get('clickTrackingParams')
 658         return cls._build_api_continuation_query(continuation, ctp)
 659
 660     @classmethod
 661     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 662         if isinstance(continuation_ep, dict):
 663             continuation = try_get(
 664                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 665             if not continuation:
 666                 return
 667             ctp = continuation_ep.get('clickTrackingParams')
 668             return cls._build_api_continuation_query(continuation, ctp)
 669
 670     @classmethod
 671     def _extract_continuation(cls, renderer):
 672         next_continuation = cls._extract_next_continuation_data(renderer)
 673         if next_continuation:
 674             return next_continuation
 675
 676         return traverse_obj(renderer, (
 677             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 678             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 679         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 680
 681     @classmethod
 682     def _extract_alerts(cls, data):
 683         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 684             if not isinstance(alert_dict, dict):
 685                 continue
 686             for alert in alert_dict.values():
 687                 alert_type = alert.get('type')
 688                 if not alert_type:
 689                     continue
 690                 message = cls._get_text(alert, 'text')
 691                 if message:
 692                     yield alert_type, message
 693
 694     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 695         errors = []
 696         warnings = []
 697         for alert_type, alert_message in alerts:
 698             if alert_type.lower() == 'error' and fatal:
 699                 errors.append([alert_type, alert_message])
 700             else:
 701                 warnings.append([alert_type, alert_message])
 702
 703         for alert_type, alert_message in (warnings + errors[:-1]):
 704             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 705         if errors:
 706             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 707
 708     def _extract_and_report_alerts(self, data, *args, **kwargs):
 709         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 710
 711     def _extract_badges(self, renderer: dict):
 712         privacy_icon_map = {
 713             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 714             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 715             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 716         }
 717
 718         badge_style_map = {
 719             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 720             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 721             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 722         }
 723
 724         label_map = {
 725             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 726             'private': BadgeType.AVAILABILITY_PRIVATE,
 727             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 728             'live': BadgeType.LIVE_NOW,
 729             'premium': BadgeType.AVAILABILITY_PREMIUM
 730         }
 731
 732         badges = []
 733         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer'), default=[]):
 734             badge_type = (
 735                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 736                 or badge_style_map.get(traverse_obj(badge, 'style'))
 737             )
 738             if badge_type:
 739                 badges.append({'type': badge_type})
 740                 continue
 741
 742             # fallback, won't work in some languages
 743             label = traverse_obj(badge, 'label', expected_type=str, default='')
 744             for match, label_badge_type in label_map.items():
 745                 if match in label.lower():
 746                     badges.append({'type': badge_type})
 747                     continue
 748
 749         return badges
 750
 751     @staticmethod
 752     def _has_badge(badges, badge_type):
 753         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 754
 755     @staticmethod
 756     def _get_text(data, *path_list, max_runs=None):
 757         for path in path_list or [None]:
 758             if path is None:
 759                 obj = [data]
 760             else:
 761                 obj = traverse_obj(data, path, default=[])
 762                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 763                     obj = [obj]
 764             for item in obj:
 765                 text = try_get(item, lambda x: x['simpleText'], str)
 766                 if text:
 767                     return text
 768                 runs = try_get(item, lambda x: x['runs'], list) or []
 769                 if not runs and isinstance(item, list):
 770                     runs = item
 771
 772                 runs = runs[:min(len(runs), max_runs or len(runs))]
 773                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 774                 if text:
 775                     return text
 776
 777     def _get_count(self, data, *path_list):
 778         count_text = self._get_text(data, *path_list) or ''
 779         count = parse_count(count_text)
 780         if count is None:
 781             count = str_to_int(
 782                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 783         return count
 784
 785     @staticmethod
 786     def _extract_thumbnails(data, *path_list):
 787         """
 788         Extract thumbnails from thumbnails dict
 789         @param path_list: path list to level that contains 'thumbnails' key
 790         """
 791         thumbnails = []
 792         for path in path_list or [()]:
 793             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 794                 thumbnail_url = url_or_none(thumbnail.get('url'))
 795                 if not thumbnail_url:
 796                     continue
 797                 # Sometimes youtube gives a wrong thumbnail URL. See:
 798                 # https://github.com/yt-dlp/yt-dlp/issues/233
 799                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 800                 if 'maxresdefault' in thumbnail_url:
 801                     thumbnail_url = thumbnail_url.split('?')[0]
 802                 thumbnails.append({
 803                     'url': thumbnail_url,
 804                     'height': int_or_none(thumbnail.get('height')),
 805                     'width': int_or_none(thumbnail.get('width')),
 806                 })
 807         return thumbnails
 808
 809     @staticmethod
 810     def extract_relative_time(relative_time_text):
 811         """
 812         Extracts a relative time from string and converts to dt object
 813         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 814         """
 815         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 816         if mobj:
 817             start = mobj.group('start')
 818             if start:
 819                 return datetime_from_str(start)
 820             try:
 821                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 822             except ValueError:
 823                 return None
 824
 825     def _parse_time_text(self, text):
 826         if not text:
 827             return
 828         dt = self.extract_relative_time(text)
 829         timestamp = None
 830         if isinstance(dt, datetime.datetime):
 831             timestamp = calendar.timegm(dt.timetuple())
 832
 833         if timestamp is None:
 834             timestamp = (
 835                 unified_timestamp(text) or unified_timestamp(
 836                     self._search_regex(
 837                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 838                         text.lower(), 'time text', default=None)))
 839
 840         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 841             self.report_warning(
 842                 f'Cannot parse localized time text "{text}"', only_once=True)
 843         return timestamp
 844
 845     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 846                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 847                           default_client='web'):
 848         for retry in self.RetryManager():
 849             try:
 850                 response = self._call_api(
 851                     ep=ep, fatal=True, headers=headers,
 852                     video_id=item_id, query=query, note=note,
 853                     context=self._extract_context(ytcfg, default_client),
 854                     api_key=self._extract_api_key(ytcfg, default_client),
 855                     api_hostname=api_hostname, default_client=default_client)
 856             except ExtractorError as e:
 857                 if not isinstance(e.cause, network_exceptions):
 858                     return self._error_or_warning(e, fatal=fatal)
 859                 elif not isinstance(e.cause, urllib.error.HTTPError):
 860                     retry.error = e
 861                     continue
 862
 863                 first_bytes = e.cause.read(512)
 864                 if not is_html(first_bytes):
 865                     yt_error = try_get(
 866                         self._parse_json(
 867                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 868                         lambda x: x['error']['message'], str)
 869                     if yt_error:
 870                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 871                 # Downloading page may result in intermittent 5xx HTTP error
 872                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 873                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 874                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 875                 if e.cause.code not in (403, 429):
 876                     retry.error = e
 877                     continue
 878                 return self._error_or_warning(e, fatal=fatal)
 879
 880             try:
 881                 self._extract_and_report_alerts(response, only_once=True)
 882             except ExtractorError as e:
 883                 # YouTube servers may return errors we want to retry on in a 200 OK response
 884                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 885                 if 'unknown error' in e.msg.lower():
 886                     retry.error = e
 887                     continue
 888                 return self._error_or_warning(e, fatal=fatal)
 889             # Youtube sometimes sends incomplete data
 890             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 891             if not traverse_obj(response, *variadic(check_get_keys)):
 892                 retry.error = ExtractorError('Incomplete data received', expected=True)
 893                 continue
 894
 895             return response
 896
 897     @staticmethod
 898     def is_music_url(url):
 899         return re.match(r'https?://music\.youtube\.com/', url) is not None
 900
 901     def _extract_video(self, renderer):
 902         video_id = renderer.get('videoId')
 903         title = self._get_text(renderer, 'title')
 904         description = self._get_text(renderer, 'descriptionSnippet')
 905         duration = parse_duration(self._get_text(
 906             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 907         if duration is None:
 908             duration = parse_duration(self._search_regex(
 909                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 910                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 911                 video_id, default=None, group='duration'))
 912
 913         view_count = self._get_count(renderer, 'viewCountText')
 914
 915         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 916         channel_id = traverse_obj(
 917             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 918             expected_type=str, get_all=False)
 919         time_text = self._get_text(renderer, 'publishedTimeText') or ''
 920         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 921         overlay_style = traverse_obj(
 922             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 923             get_all=False, expected_type=str)
 924         badges = self._extract_badges(renderer)
 925         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 926         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 927             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 928             expected_type=str)) or ''
 929         url = f'https://www.youtube.com/watch?v={video_id}'
 930         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 931             url = f'https://www.youtube.com/shorts/{video_id}'
 932
 933         return {
 934             '_type': 'url',
 935             'ie_key': YoutubeIE.ie_key(),
 936             'id': video_id,
 937             'url': url,
 938             'title': title,
 939             'description': description,
 940             'duration': duration,
 941             'view_count': view_count,
 942             'uploader': uploader,
 943             'channel_id': channel_id,
 944             'thumbnails': thumbnails,
 945             'upload_date': (strftime_or_none(self._parse_time_text(time_text), '%Y%m%d')
 946                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 947                             else None),
 948             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 949                             else 'was_live' if 'streamed' in time_text.lower()
 950                             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
 951                             else None),
 952             'release_timestamp': scheduled_timestamp,
 953             'availability':
 954                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
 955                 else self._availability(
 956                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
 957                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
 958                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
 959                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None)
 960         }
 961
 962
 963 class YoutubeIE(YoutubeBaseInfoExtractor):
 964     IE_DESC = 'YouTube'
 965     _VALID_URL = r"""(?x)^
 966                      (
 967                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 968                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 969                             (?:www\.)?deturl\.com/www\.youtube\.com|
 970                             (?:www\.)?pwnyoutube\.com|
 971                             (?:www\.)?hooktube\.com|
 972                             (?:www\.)?yourepeat\.com|
 973                             tube\.majestyc\.net|
 974                             %(invidious)s|
 975                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 976                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 977                          (?:                                                  # the various things that can precede the ID:
 978                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 979                              |(?:                                             # or the v= param in all its forms
 980                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 981                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 982                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 983                                  v=
 984                              )
 985                          ))
 986                          |(?:
 987                             youtu\.be|                                        # just youtu.be/xxxx
 988                             vid\.plus|                                        # or vid.plus/xxxx
 989                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 990                             %(invidious)s
 991                          )/
 992                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 993                          )
 994                      )?                                                       # all until now is optional -> you can pass the naked ID
 995                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 996                      (?(1).+)?                                                # if we found the ID, everything can follow
 997                      (?:\#|$)""" % {
 998         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 999     }
1000     _EMBED_REGEX = [
1001         r'''(?x)
1002             (?:
1003                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1004                 data-video-url=|
1005                 <embed[^>]+?src=|
1006                 embedSWF\(?:\s*|
1007                 <object[^>]+data=|
1008                 new\s+SWFObject\(
1009             )
1010             (["\'])
1011                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1012                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1013             \1''',
1014         # https://wordpress.org/plugins/lazy-load-for-videos/
1015         r'''(?xs)
1016             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1017             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1018     ]
1019
1020     _PLAYER_INFO_RE = (
1021         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1022         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1023         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1024     )
1025     _formats = {
1026         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1027         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1028         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1029         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1030         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1031         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1032         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1033         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1034         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1035         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1036         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1037         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1038         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1039         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1040         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1041         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1042         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1043         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1044
1045
1046         # 3D videos
1047         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1048         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1049         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1050         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1051         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1052         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1053         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1054
1055         # Apple HTTP Live Streaming
1056         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1057         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1058         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1059         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1060         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1061         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1062         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1063         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1064
1065         # DASH mp4 video
1066         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1067         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1068         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1069         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1070         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1071         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1072         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1073         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1074         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1075         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1076         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1077         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1078
1079         # Dash mp4 audio
1080         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1081         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1082         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1083         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1084         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1085         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1086         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1087
1088         # Dash webm
1089         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1090         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1091         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1092         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1093         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1094         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1095         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1096         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1097         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1098         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1099         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1100         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1101         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1102         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1103         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1104         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1105         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1106         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1107         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1108         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1109         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1110         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1111
1112         # Dash webm audio
1113         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1114         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1115
1116         # Dash webm audio with opus inside
1117         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1118         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1119         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1120
1121         # RTMP (unnamed)
1122         '_rtmp': {'protocol': 'rtmp'},
1123
1124         # av01 video only formats sometimes served with "unknown" codecs
1125         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1126         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1127         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1128         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1129         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1130         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1131         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1132         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1133     }
1134     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1135
1136     _GEO_BYPASS = False
1137
1138     IE_NAME = 'youtube'
1139     _TESTS = [
1140         {
1141             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1142             'info_dict': {
1143                 'id': 'BaW_jenozKc',
1144                 'ext': 'mp4',
1145                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1146                 'uploader': 'Philipp Hagemeister',
1147                 'uploader_id': 'phihag',
1148                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1149                 'channel': 'Philipp Hagemeister',
1150                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1151                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1152                 'upload_date': '20121002',
1153                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1154                 'categories': ['Science & Technology'],
1155                 'tags': ['youtube-dl'],
1156                 'duration': 10,
1157                 'view_count': int,
1158                 'like_count': int,
1159                 'availability': 'public',
1160                 'playable_in_embed': True,
1161                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1162                 'live_status': 'not_live',
1163                 'age_limit': 0,
1164                 'start_time': 1,
1165                 'end_time': 9,
1166                 'comment_count': int,
1167                 'channel_follower_count': int
1168             }
1169         },
1170         {
1171             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1172             'note': 'Embed-only video (#1746)',
1173             'info_dict': {
1174                 'id': 'yZIXLfi8CZQ',
1175                 'ext': 'mp4',
1176                 'upload_date': '20120608',
1177                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1178                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1179                 'uploader': 'SET India',
1180                 'uploader_id': 'setindia',
1181                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1182                 'age_limit': 18,
1183             },
1184             'skip': 'Private video',
1185         },
1186         {
1187             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1188             'note': 'Use the first video ID in the URL',
1189             'info_dict': {
1190                 'id': 'BaW_jenozKc',
1191                 'ext': 'mp4',
1192                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1193                 'uploader': 'Philipp Hagemeister',
1194                 'uploader_id': 'phihag',
1195                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1196                 'channel': 'Philipp Hagemeister',
1197                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1198                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1199                 'upload_date': '20121002',
1200                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1201                 'categories': ['Science & Technology'],
1202                 'tags': ['youtube-dl'],
1203                 'duration': 10,
1204                 'view_count': int,
1205                 'like_count': int,
1206                 'availability': 'public',
1207                 'playable_in_embed': True,
1208                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1209                 'live_status': 'not_live',
1210                 'age_limit': 0,
1211                 'comment_count': int,
1212                 'channel_follower_count': int
1213             },
1214             'params': {
1215                 'skip_download': True,
1216             },
1217         },
1218         {
1219             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1220             'note': '256k DASH audio (format 141) via DASH manifest',
1221             'info_dict': {
1222                 'id': 'a9LDPn-MO4I',
1223                 'ext': 'm4a',
1224                 'upload_date': '20121002',
1225                 'uploader_id': '8KVIDEO',
1226                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1227                 'description': '',
1228                 'uploader': '8KVIDEO',
1229                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1230             },
1231             'params': {
1232                 'youtube_include_dash_manifest': True,
1233                 'format': '141',
1234             },
1235             'skip': 'format 141 not served anymore',
1236         },
1237         # DASH manifest with encrypted signature
1238         {
1239             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1240             'info_dict': {
1241                 'id': 'IB3lcPjvWLA',
1242                 'ext': 'm4a',
1243                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1244                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1245                 'duration': 244,
1246                 'uploader': 'AfrojackVEVO',
1247                 'uploader_id': 'AfrojackVEVO',
1248                 'upload_date': '20131011',
1249                 'abr': 129.495,
1250                 'like_count': int,
1251                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1252                 'playable_in_embed': True,
1253                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1254                 'view_count': int,
1255                 'track': 'The Spark',
1256                 'live_status': 'not_live',
1257                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1258                 'channel': 'Afrojack',
1259                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1260                 'tags': 'count:19',
1261                 'availability': 'public',
1262                 'categories': ['Music'],
1263                 'age_limit': 0,
1264                 'alt_title': 'The Spark',
1265                 'channel_follower_count': int
1266             },
1267             'params': {
1268                 'youtube_include_dash_manifest': True,
1269                 'format': '141/bestaudio[ext=m4a]',
1270             },
1271         },
1272         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1273         {
1274             'note': 'Embed allowed age-gate video',
1275             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1276             'info_dict': {
1277                 'id': 'HtVdAasjOgU',
1278                 'ext': 'mp4',
1279                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1280                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1281                 'duration': 142,
1282                 'uploader': 'The Witcher',
1283                 'uploader_id': 'WitcherGame',
1284                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1285                 'upload_date': '20140605',
1286                 'age_limit': 18,
1287                 'categories': ['Gaming'],
1288                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1289                 'availability': 'needs_auth',
1290                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1291                 'like_count': int,
1292                 'channel': 'The Witcher',
1293                 'live_status': 'not_live',
1294                 'tags': 'count:17',
1295                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1296                 'playable_in_embed': True,
1297                 'view_count': int,
1298                 'channel_follower_count': int
1299             },
1300         },
1301         {
1302             'note': 'Age-gate video with embed allowed in public site',
1303             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1304             'info_dict': {
1305                 'id': 'HsUATh_Nc2U',
1306                 'ext': 'mp4',
1307                 'title': 'Godzilla 2 (Official Video)',
1308                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1309                 'upload_date': '20200408',
1310                 'uploader_id': 'FlyingKitty900',
1311                 'uploader': 'FlyingKitty',
1312                 'age_limit': 18,
1313                 'availability': 'needs_auth',
1314                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1315                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1316                 'channel': 'FlyingKitty',
1317                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1318                 'view_count': int,
1319                 'categories': ['Entertainment'],
1320                 'live_status': 'not_live',
1321                 'tags': ['Flyingkitty', 'godzilla 2'],
1322                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1323                 'like_count': int,
1324                 'duration': 177,
1325                 'playable_in_embed': True,
1326                 'channel_follower_count': int
1327             },
1328         },
1329         {
1330             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1331             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1332             'info_dict': {
1333                 'id': 'Tq92D6wQ1mg',
1334                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1335                 'ext': 'mp4',
1336                 'upload_date': '20191228',
1337                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1338                 'uploader': 'Projekt Melody',
1339                 'description': 'md5:17eccca93a786d51bc67646756894066',
1340                 'age_limit': 18,
1341                 'like_count': int,
1342                 'availability': 'needs_auth',
1343                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1344                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1345                 'view_count': int,
1346                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1347                 'channel': 'Projekt Melody',
1348                 'live_status': 'not_live',
1349                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1350                 'playable_in_embed': True,
1351                 'categories': ['Entertainment'],
1352                 'duration': 106,
1353                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1354                 'comment_count': int,
1355                 'channel_follower_count': int
1356             },
1357         },
1358         {
1359             'note': 'Non-Agegated non-embeddable video',
1360             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1361             'info_dict': {
1362                 'id': 'MeJVWBSsPAY',
1363                 'ext': 'mp4',
1364                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1365                 'uploader': 'Herr Lurik',
1366                 'uploader_id': 'st3in234',
1367                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1368                 'upload_date': '20130730',
1369                 'track': 'Such mich find mich',
1370                 'age_limit': 0,
1371                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1372                 'like_count': int,
1373                 'playable_in_embed': False,
1374                 'creator': 'OOMPH!',
1375                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1376                 'view_count': int,
1377                 'alt_title': 'Such mich find mich',
1378                 'duration': 210,
1379                 'channel': 'Herr Lurik',
1380                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1381                 'categories': ['Music'],
1382                 'availability': 'public',
1383                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1384                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1385                 'live_status': 'not_live',
1386                 'artist': 'OOMPH!',
1387                 'channel_follower_count': int
1388             },
1389         },
1390         {
1391             'note': 'Non-bypassable age-gated video',
1392             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1393             'only_matching': True,
1394         },
1395         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1396         # YouTube Red ad is not captured for creator
1397         {
1398             'url': '__2ABJjxzNo',
1399             'info_dict': {
1400                 'id': '__2ABJjxzNo',
1401                 'ext': 'mp4',
1402                 'duration': 266,
1403                 'upload_date': '20100430',
1404                 'uploader_id': 'deadmau5',
1405                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1406                 'creator': 'deadmau5',
1407                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1408                 'uploader': 'deadmau5',
1409                 'title': 'Deadmau5 - Some Chords (HD)',
1410                 'alt_title': 'Some Chords',
1411                 'availability': 'public',
1412                 'tags': 'count:14',
1413                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1414                 'view_count': int,
1415                 'live_status': 'not_live',
1416                 'channel': 'deadmau5',
1417                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1418                 'like_count': int,
1419                 'track': 'Some Chords',
1420                 'artist': 'deadmau5',
1421                 'playable_in_embed': True,
1422                 'age_limit': 0,
1423                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1424                 'categories': ['Music'],
1425                 'album': 'Some Chords',
1426                 'channel_follower_count': int
1427             },
1428             'expected_warnings': [
1429                 'DASH manifest missing',
1430             ]
1431         },
1432         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1433         {
1434             'url': 'lqQg6PlCWgI',
1435             'info_dict': {
1436                 'id': 'lqQg6PlCWgI',
1437                 'ext': 'mp4',
1438                 'duration': 6085,
1439                 'upload_date': '20150827',
1440                 'uploader_id': 'olympic',
1441                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1442                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1443                 'uploader': 'Olympics',
1444                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1445                 'like_count': int,
1446                 'release_timestamp': 1343767800,
1447                 'playable_in_embed': True,
1448                 'categories': ['Sports'],
1449                 'release_date': '20120731',
1450                 'channel': 'Olympics',
1451                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1452                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1453                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1454                 'age_limit': 0,
1455                 'availability': 'public',
1456                 'live_status': 'was_live',
1457                 'view_count': int,
1458                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1459                 'channel_follower_count': int
1460             },
1461             'params': {
1462                 'skip_download': 'requires avconv',
1463             }
1464         },
1465         # Non-square pixels
1466         {
1467             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1468             'info_dict': {
1469                 'id': '_b-2C3KPAM0',
1470                 'ext': 'mp4',
1471                 'stretched_ratio': 16 / 9.,
1472                 'duration': 85,
1473                 'upload_date': '20110310',
1474                 'uploader_id': 'AllenMeow',
1475                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1476                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1477                 'uploader': '孫ᄋᄅ',
1478                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1479                 'playable_in_embed': True,
1480                 'channel': '孫ᄋᄅ',
1481                 'age_limit': 0,
1482                 'tags': 'count:11',
1483                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1484                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1485                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1486                 'view_count': int,
1487                 'categories': ['People & Blogs'],
1488                 'like_count': int,
1489                 'live_status': 'not_live',
1490                 'availability': 'unlisted',
1491                 'comment_count': int,
1492                 'channel_follower_count': int
1493             },
1494         },
1495         # url_encoded_fmt_stream_map is empty string
1496         {
1497             'url': 'qEJwOuvDf7I',
1498             'info_dict': {
1499                 'id': 'qEJwOuvDf7I',
1500                 'ext': 'webm',
1501                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1502                 'description': '',
1503                 'upload_date': '20150404',
1504                 'uploader_id': 'spbelect',
1505                 'uploader': 'Наблюдатели Петербурга',
1506             },
1507             'params': {
1508                 'skip_download': 'requires avconv',
1509             },
1510             'skip': 'This live event has ended.',
1511         },
1512         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1513         {
1514             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1515             'info_dict': {
1516                 'id': 'FIl7x6_3R5Y',
1517                 'ext': 'webm',
1518                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1519                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1520                 'duration': 220,
1521                 'upload_date': '20150625',
1522                 'uploader_id': 'dorappi2000',
1523                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1524                 'uploader': 'dorappi2000',
1525                 'formats': 'mincount:31',
1526             },
1527             'skip': 'not actual anymore',
1528         },
1529         # DASH manifest with segment_list
1530         {
1531             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1532             'md5': '8ce563a1d667b599d21064e982ab9e31',
1533             'info_dict': {
1534                 'id': 'CsmdDsKjzN8',
1535                 'ext': 'mp4',
1536                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1537                 'uploader': 'Airtek',
1538                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1539                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1540                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1541             },
1542             'params': {
1543                 'youtube_include_dash_manifest': True,
1544                 'format': '135',  # bestvideo
1545             },
1546             'skip': 'This live event has ended.',
1547         },
1548         {
1549             # Multifeed videos (multiple cameras), URL is for Main Camera
1550             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1551             'info_dict': {
1552                 'id': 'jvGDaLqkpTg',
1553                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1554                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1555             },
1556             'playlist': [{
1557                 'info_dict': {
1558                     'id': 'jvGDaLqkpTg',
1559                     'ext': 'mp4',
1560                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1561                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1562                     'duration': 10643,
1563                     'upload_date': '20161111',
1564                     'uploader': 'Team PGP',
1565                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1566                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1567                 },
1568             }, {
1569                 'info_dict': {
1570                     'id': '3AKt1R1aDnw',
1571                     'ext': 'mp4',
1572                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1573                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1574                     'duration': 10991,
1575                     'upload_date': '20161111',
1576                     'uploader': 'Team PGP',
1577                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1578                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1579                 },
1580             }, {
1581                 'info_dict': {
1582                     'id': 'RtAMM00gpVc',
1583                     'ext': 'mp4',
1584                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1585                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1586                     'duration': 10995,
1587                     'upload_date': '20161111',
1588                     'uploader': 'Team PGP',
1589                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1590                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1591                 },
1592             }, {
1593                 'info_dict': {
1594                     'id': '6N2fdlP3C5U',
1595                     'ext': 'mp4',
1596                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1597                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1598                     'duration': 10990,
1599                     'upload_date': '20161111',
1600                     'uploader': 'Team PGP',
1601                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1602                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1603                 },
1604             }],
1605             'params': {
1606                 'skip_download': True,
1607             },
1608             'skip': 'Not multifeed anymore',
1609         },
1610         {
1611             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1612             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1613             'info_dict': {
1614                 'id': 'gVfLd0zydlo',
1615                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1616             },
1617             'playlist_count': 2,
1618             'skip': 'Not multifeed anymore',
1619         },
1620         {
1621             'url': 'https://vid.plus/FlRa-iH7PGw',
1622             'only_matching': True,
1623         },
1624         {
1625             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1626             'only_matching': True,
1627         },
1628         {
1629             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1630             # Also tests cut-off URL expansion in video description (see
1631             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1632             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1633             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1634             'info_dict': {
1635                 'id': 'lsguqyKfVQg',
1636                 'ext': 'mp4',
1637                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1638                 'alt_title': 'Dark Walk',
1639                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1640                 'duration': 133,
1641                 'upload_date': '20151119',
1642                 'uploader_id': 'IronSoulElf',
1643                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1644                 'uploader': 'IronSoulElf',
1645                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1646                 'track': 'Dark Walk',
1647                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1648                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1649                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1650                 'categories': ['Film & Animation'],
1651                 'view_count': int,
1652                 'live_status': 'not_live',
1653                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1654                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1655                 'tags': 'count:13',
1656                 'availability': 'public',
1657                 'channel': 'IronSoulElf',
1658                 'playable_in_embed': True,
1659                 'like_count': int,
1660                 'age_limit': 0,
1661                 'channel_follower_count': int
1662             },
1663             'params': {
1664                 'skip_download': True,
1665             },
1666         },
1667         {
1668             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1669             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1670             'only_matching': True,
1671         },
1672         {
1673             # Video with yt:stretch=17:0
1674             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1675             'info_dict': {
1676                 'id': 'Q39EVAstoRM',
1677                 'ext': 'mp4',
1678                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1679                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1680                 'upload_date': '20151107',
1681                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1682                 'uploader': 'CH GAMER DROID',
1683             },
1684             'params': {
1685                 'skip_download': True,
1686             },
1687             'skip': 'This video does not exist.',
1688         },
1689         {
1690             # Video with incomplete 'yt:stretch=16:'
1691             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1692             'only_matching': True,
1693         },
1694         {
1695             # Video licensed under Creative Commons
1696             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1697             'info_dict': {
1698                 'id': 'M4gD1WSo5mA',
1699                 'ext': 'mp4',
1700                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1701                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1702                 'duration': 721,
1703                 'upload_date': '20150128',
1704                 'uploader_id': 'BerkmanCenter',
1705                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1706                 'uploader': 'The Berkman Klein Center for Internet & Society',
1707                 'license': 'Creative Commons Attribution license (reuse allowed)',
1708                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1709                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1710                 'like_count': int,
1711                 'age_limit': 0,
1712                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1713                 'channel': 'The Berkman Klein Center for Internet & Society',
1714                 'availability': 'public',
1715                 'view_count': int,
1716                 'categories': ['Education'],
1717                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1718                 'live_status': 'not_live',
1719                 'playable_in_embed': True,
1720                 'comment_count': int,
1721                 'channel_follower_count': int
1722             },
1723             'params': {
1724                 'skip_download': True,
1725             },
1726         },
1727         {
1728             # Channel-like uploader_url
1729             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1730             'info_dict': {
1731                 'id': 'eQcmzGIKrzg',
1732                 'ext': 'mp4',
1733                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1734                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1735                 'duration': 4060,
1736                 'upload_date': '20151120',
1737                 'uploader': 'Bernie Sanders',
1738                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1739                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1740                 'license': 'Creative Commons Attribution license (reuse allowed)',
1741                 'playable_in_embed': True,
1742                 'tags': 'count:12',
1743                 'like_count': int,
1744                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1745                 'age_limit': 0,
1746                 'availability': 'public',
1747                 'categories': ['News & Politics'],
1748                 'channel': 'Bernie Sanders',
1749                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1750                 'view_count': int,
1751                 'live_status': 'not_live',
1752                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1753                 'comment_count': int,
1754                 'channel_follower_count': int
1755             },
1756             'params': {
1757                 'skip_download': True,
1758             },
1759         },
1760         {
1761             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1762             'only_matching': True,
1763         },
1764         {
1765             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1766             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1767             'only_matching': True,
1768         },
1769         {
1770             # Rental video preview
1771             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1772             'info_dict': {
1773                 'id': 'uGpuVWrhIzE',
1774                 'ext': 'mp4',
1775                 'title': 'Piku - Trailer',
1776                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1777                 'upload_date': '20150811',
1778                 'uploader': 'FlixMatrix',
1779                 'uploader_id': 'FlixMatrixKaravan',
1780                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1781                 'license': 'Standard YouTube License',
1782             },
1783             'params': {
1784                 'skip_download': True,
1785             },
1786             'skip': 'This video is not available.',
1787         },
1788         {
1789             # YouTube Red video with episode data
1790             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1791             'info_dict': {
1792                 'id': 'iqKdEhx-dD4',
1793                 'ext': 'mp4',
1794                 'title': 'Isolation - Mind Field (Ep 1)',
1795                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1796                 'duration': 2085,
1797                 'upload_date': '20170118',
1798                 'uploader': 'Vsauce',
1799                 'uploader_id': 'Vsauce',
1800                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1801                 'series': 'Mind Field',
1802                 'season_number': 1,
1803                 'episode_number': 1,
1804                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1805                 'tags': 'count:12',
1806                 'view_count': int,
1807                 'availability': 'public',
1808                 'age_limit': 0,
1809                 'channel': 'Vsauce',
1810                 'episode': 'Episode 1',
1811                 'categories': ['Entertainment'],
1812                 'season': 'Season 1',
1813                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1814                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1815                 'like_count': int,
1816                 'playable_in_embed': True,
1817                 'live_status': 'not_live',
1818                 'channel_follower_count': int
1819             },
1820             'params': {
1821                 'skip_download': True,
1822             },
1823             'expected_warnings': [
1824                 'Skipping DASH manifest',
1825             ],
1826         },
1827         {
1828             # The following content has been identified by the YouTube community
1829             # as inappropriate or offensive to some audiences.
1830             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1831             'info_dict': {
1832                 'id': '6SJNVb0GnPI',
1833                 'ext': 'mp4',
1834                 'title': 'Race Differences in Intelligence',
1835                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1836                 'duration': 965,
1837                 'upload_date': '20140124',
1838                 'uploader': 'New Century Foundation',
1839                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1840                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1841             },
1842             'params': {
1843                 'skip_download': True,
1844             },
1845             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1846         },
1847         {
1848             # itag 212
1849             'url': '1t24XAntNCY',
1850             'only_matching': True,
1851         },
1852         {
1853             # geo restricted to JP
1854             'url': 'sJL6WA-aGkQ',
1855             'only_matching': True,
1856         },
1857         {
1858             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1859             'only_matching': True,
1860         },
1861         {
1862             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1863             'only_matching': True,
1864         },
1865         {
1866             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1867             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1868             'only_matching': True,
1869         },
1870         {
1871             # DRM protected
1872             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1873             'only_matching': True,
1874         },
1875         {
1876             # Video with unsupported adaptive stream type formats
1877             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1878             'info_dict': {
1879                 'id': 'Z4Vy8R84T1U',
1880                 'ext': 'mp4',
1881                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1882                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1883                 'duration': 433,
1884                 'upload_date': '20130923',
1885                 'uploader': 'Amelia Putri Harwita',
1886                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1887                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1888                 'formats': 'maxcount:10',
1889             },
1890             'params': {
1891                 'skip_download': True,
1892                 'youtube_include_dash_manifest': False,
1893             },
1894             'skip': 'not actual anymore',
1895         },
1896         {
1897             # Youtube Music Auto-generated description
1898             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1899             'info_dict': {
1900                 'id': 'MgNrAu2pzNs',
1901                 'ext': 'mp4',
1902                 'title': 'Voyeur Girl',
1903                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1904                 'upload_date': '20190312',
1905                 'uploader': 'Stephen - Topic',
1906                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1907                 'artist': 'Stephen',
1908                 'track': 'Voyeur Girl',
1909                 'album': 'it\'s too much love to know my dear',
1910                 'release_date': '20190313',
1911                 'release_year': 2019,
1912                 'alt_title': 'Voyeur Girl',
1913                 'view_count': int,
1914                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1915                 'playable_in_embed': True,
1916                 'like_count': int,
1917                 'categories': ['Music'],
1918                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1919                 'channel': 'Stephen',
1920                 'availability': 'public',
1921                 'creator': 'Stephen',
1922                 'duration': 169,
1923                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1924                 'age_limit': 0,
1925                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1926                 'tags': 'count:11',
1927                 'live_status': 'not_live',
1928                 'channel_follower_count': int
1929             },
1930             'params': {
1931                 'skip_download': True,
1932             },
1933         },
1934         {
1935             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1936             'only_matching': True,
1937         },
1938         {
1939             # invalid -> valid video id redirection
1940             'url': 'DJztXj2GPfl',
1941             'info_dict': {
1942                 'id': 'DJztXj2GPfk',
1943                 'ext': 'mp4',
1944                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1945                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1946                 'upload_date': '20090125',
1947                 'uploader': 'Prochorowka',
1948                 'uploader_id': 'Prochorowka',
1949                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1950                 'artist': 'Panjabi MC',
1951                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1952                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1953             },
1954             'params': {
1955                 'skip_download': True,
1956             },
1957             'skip': 'Video unavailable',
1958         },
1959         {
1960             # empty description results in an empty string
1961             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1962             'info_dict': {
1963                 'id': 'x41yOUIvK2k',
1964                 'ext': 'mp4',
1965                 'title': 'IMG 3456',
1966                 'description': '',
1967                 'upload_date': '20170613',
1968                 'uploader_id': 'ElevageOrVert',
1969                 'uploader': 'ElevageOrVert',
1970                 'view_count': int,
1971                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1972                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1973                 'like_count': int,
1974                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1975                 'tags': [],
1976                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1977                 'availability': 'public',
1978                 'age_limit': 0,
1979                 'categories': ['Pets & Animals'],
1980                 'duration': 7,
1981                 'playable_in_embed': True,
1982                 'live_status': 'not_live',
1983                 'channel': 'ElevageOrVert',
1984                 'channel_follower_count': int
1985             },
1986             'params': {
1987                 'skip_download': True,
1988             },
1989         },
1990         {
1991             # with '};' inside yt initial data (see [1])
1992             # see [2] for an example with '};' inside ytInitialPlayerResponse
1993             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1994             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1995             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1996             'info_dict': {
1997                 'id': 'CHqg6qOn4no',
1998                 'ext': 'mp4',
1999                 'title': 'Part 77   Sort a list of simple types in c#',
2000                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2001                 'upload_date': '20130831',
2002                 'uploader_id': 'kudvenkat',
2003                 'uploader': 'kudvenkat',
2004                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2005                 'like_count': int,
2006                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2007                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2008                 'live_status': 'not_live',
2009                 'categories': ['Education'],
2010                 'availability': 'public',
2011                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2012                 'tags': 'count:12',
2013                 'playable_in_embed': True,
2014                 'age_limit': 0,
2015                 'view_count': int,
2016                 'duration': 522,
2017                 'channel': 'kudvenkat',
2018                 'comment_count': int,
2019                 'channel_follower_count': int
2020             },
2021             'params': {
2022                 'skip_download': True,
2023             },
2024         },
2025         {
2026             # another example of '};' in ytInitialData
2027             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2028             'only_matching': True,
2029         },
2030         {
2031             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2032             'only_matching': True,
2033         },
2034         {
2035             # https://github.com/ytdl-org/youtube-dl/pull/28094
2036             'url': 'OtqTfy26tG0',
2037             'info_dict': {
2038                 'id': 'OtqTfy26tG0',
2039                 'ext': 'mp4',
2040                 'title': 'Burn Out',
2041                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2042                 'upload_date': '20141120',
2043                 'uploader': 'The Cinematic Orchestra - Topic',
2044                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2045                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2046                 'artist': 'The Cinematic Orchestra',
2047                 'track': 'Burn Out',
2048                 'album': 'Every Day',
2049                 'like_count': int,
2050                 'live_status': 'not_live',
2051                 'alt_title': 'Burn Out',
2052                 'duration': 614,
2053                 'age_limit': 0,
2054                 'view_count': int,
2055                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2056                 'creator': 'The Cinematic Orchestra',
2057                 'channel': 'The Cinematic Orchestra',
2058                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2059                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2060                 'availability': 'public',
2061                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2062                 'categories': ['Music'],
2063                 'playable_in_embed': True,
2064                 'channel_follower_count': int
2065             },
2066             'params': {
2067                 'skip_download': True,
2068             },
2069         },
2070         {
2071             # controversial video, only works with bpctr when authenticated with cookies
2072             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2073             'only_matching': True,
2074         },
2075         {
2076             # controversial video, requires bpctr/contentCheckOk
2077             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2078             'info_dict': {
2079                 'id': 'SZJvDhaSDnc',
2080                 'ext': 'mp4',
2081                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2082                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2083                 'uploader': 'CBS Mornings',
2084                 'uploader_id': 'CBSThisMorning',
2085                 'upload_date': '20140716',
2086                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2087                 'duration': 170,
2088                 'categories': ['News & Politics'],
2089                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2090                 'view_count': int,
2091                 'channel': 'CBS Mornings',
2092                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2093                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2094                 'age_limit': 18,
2095                 'availability': 'needs_auth',
2096                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2097                 'like_count': int,
2098                 'live_status': 'not_live',
2099                 'playable_in_embed': True,
2100                 'channel_follower_count': int
2101             }
2102         },
2103         {
2104             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2105             'url': 'cBvYw8_A0vQ',
2106             'info_dict': {
2107                 'id': 'cBvYw8_A0vQ',
2108                 'ext': 'mp4',
2109                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2110                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2111                 'upload_date': '20201120',
2112                 'uploader': 'Walk around Japan',
2113                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2114                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2115                 'duration': 1456,
2116                 'categories': ['Travel & Events'],
2117                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2118                 'view_count': int,
2119                 'channel': 'Walk around Japan',
2120                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2121                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2122                 'age_limit': 0,
2123                 'availability': 'public',
2124                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2125                 'live_status': 'not_live',
2126                 'playable_in_embed': True,
2127                 'channel_follower_count': int
2128             },
2129             'params': {
2130                 'skip_download': True,
2131             },
2132         }, {
2133             # Has multiple audio streams
2134             'url': 'WaOKSUlf4TM',
2135             'only_matching': True
2136         }, {
2137             # Requires Premium: has format 141 when requested using YTM url
2138             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2139             'only_matching': True
2140         }, {
2141             # multiple subtitles with same lang_code
2142             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2143             'only_matching': True,
2144         }, {
2145             # Force use android client fallback
2146             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2147             'info_dict': {
2148                 'id': 'YOelRv7fMxY',
2149                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2150                 'ext': '3gp',
2151                 'upload_date': '20210624',
2152                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2153                 'uploader': 'colinfurze',
2154                 'uploader_id': 'colinfurze',
2155                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2156                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2157                 'duration': 596,
2158                 'categories': ['Entertainment'],
2159                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2160                 'view_count': int,
2161                 'channel': 'colinfurze',
2162                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2163                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2164                 'age_limit': 0,
2165                 'availability': 'public',
2166                 'like_count': int,
2167                 'live_status': 'not_live',
2168                 'playable_in_embed': True,
2169                 'channel_follower_count': int
2170             },
2171             'params': {
2172                 'format': '17',  # 3gp format available on android
2173                 'extractor_args': {'youtube': {'player_client': ['android']}},
2174             },
2175         },
2176         {
2177             # Skip download of additional client configs (remix client config in this case)
2178             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2179             'only_matching': True,
2180             'params': {
2181                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2182             },
2183         }, {
2184             # shorts
2185             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2186             'only_matching': True,
2187         }, {
2188             'note': 'Storyboards',
2189             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2190             'info_dict': {
2191                 'id': '5KLPxDtMqe8',
2192                 'ext': 'mhtml',
2193                 'format_id': 'sb0',
2194                 'title': 'Your Brain is Plastic',
2195                 'uploader_id': 'scishow',
2196                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2197                 'upload_date': '20140324',
2198                 'uploader': 'SciShow',
2199                 'like_count': int,
2200                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2201                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2202                 'view_count': int,
2203                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2204                 'playable_in_embed': True,
2205                 'tags': 'count:12',
2206                 'uploader_url': 'http://www.youtube.com/user/scishow',
2207                 'availability': 'public',
2208                 'channel': 'SciShow',
2209                 'live_status': 'not_live',
2210                 'duration': 248,
2211                 'categories': ['Education'],
2212                 'age_limit': 0,
2213                 'channel_follower_count': int
2214             }, 'params': {'format': 'mhtml', 'skip_download': True}
2215         }, {
2216             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2217             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2218             'info_dict': {
2219                 'id': '2NUZ8W2llS4',
2220                 'ext': 'mp4',
2221                 'title': 'The NP that test your phone performance 🙂',
2222                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2223                 'uploader': 'Leon Nguyen',
2224                 'uploader_id': 'VNSXIII',
2225                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2226                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2227                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2228                 'duration': 21,
2229                 'view_count': int,
2230                 'age_limit': 0,
2231                 'categories': ['Gaming'],
2232                 'tags': 'count:23',
2233                 'playable_in_embed': True,
2234                 'live_status': 'not_live',
2235                 'upload_date': '20220103',
2236                 'like_count': int,
2237                 'availability': 'public',
2238                 'channel': 'Leon Nguyen',
2239                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2240                 'comment_count': int,
2241                 'channel_follower_count': int
2242             }
2243         }, {
2244             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2245             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2246             'info_dict': {
2247                 'id': '2NUZ8W2llS4',
2248                 'ext': 'mp4',
2249                 'title': 'The NP that test your phone performance 🙂',
2250                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2251                 'uploader': 'Leon Nguyen',
2252                 'uploader_id': 'VNSXIII',
2253                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2254                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2255                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2256                 'duration': 21,
2257                 'view_count': int,
2258                 'age_limit': 0,
2259                 'categories': ['Gaming'],
2260                 'tags': 'count:23',
2261                 'playable_in_embed': True,
2262                 'live_status': 'not_live',
2263                 'upload_date': '20220102',
2264                 'like_count': int,
2265                 'availability': 'public',
2266                 'channel': 'Leon Nguyen',
2267                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2268                 'comment_count': int,
2269                 'channel_follower_count': int
2270             },
2271             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2272         }, {
2273             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2274             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2275             'info_dict': {
2276                 'id': 'mzZzzBU6lrM',
2277                 'ext': 'mp4',
2278                 'title': 'I Met GeorgeNotFound In Real Life...',
2279                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2280                 'uploader': 'Quackity',
2281                 'uploader_id': 'QuackityHQ',
2282                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2283                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2284                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2285                 'duration': 955,
2286                 'view_count': int,
2287                 'age_limit': 0,
2288                 'categories': ['Entertainment'],
2289                 'tags': 'count:26',
2290                 'playable_in_embed': True,
2291                 'live_status': 'not_live',
2292                 'release_timestamp': 1641172509,
2293                 'release_date': '20220103',
2294                 'upload_date': '20220103',
2295                 'like_count': int,
2296                 'availability': 'public',
2297                 'channel': 'Quackity',
2298                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2299                 'channel_follower_count': int
2300             }
2301         },
2302         {   # continuous livestream. Microformat upload date should be preferred.
2303             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2304             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2305             'info_dict': {
2306                 'id': 'kgx4WGK0oNU',
2307                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2308                 'ext': 'mp4',
2309                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2310                 'availability': 'public',
2311                 'age_limit': 0,
2312                 'release_timestamp': 1637975704,
2313                 'upload_date': '20210619',
2314                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2315                 'live_status': 'is_live',
2316                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2317                 'uploader': '阿鲍Abao',
2318                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2319                 'channel': 'Abao in Tokyo',
2320                 'channel_follower_count': int,
2321                 'release_date': '20211127',
2322                 'tags': 'count:39',
2323                 'categories': ['People & Blogs'],
2324                 'like_count': int,
2325                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2326                 'view_count': int,
2327                 'playable_in_embed': True,
2328                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2329             },
2330             'params': {'skip_download': True}
2331         }, {
2332             # Story. Requires specific player params to work.
2333             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2334             'info_dict': {
2335                 'id': 'vv8qTUWmulI',
2336                 'ext': 'mp4',
2337                 'availability': 'unlisted',
2338                 'view_count': int,
2339                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2340                 'upload_date': '20220526',
2341                 'categories': ['Education'],
2342                 'title': 'Story',
2343                 'channel': 'IT\'S HISTORY',
2344                 'description': '',
2345                 'uploader_id': 'BlastfromthePast',
2346                 'duration': 12,
2347                 'uploader': 'IT\'S HISTORY',
2348                 'playable_in_embed': True,
2349                 'age_limit': 0,
2350                 'live_status': 'not_live',
2351                 'tags': [],
2352                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2353                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2354                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2355             },
2356             'skip': 'stories get removed after some period of time',
2357         }, {
2358             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2359             'info_dict': {
2360                 'id': 'tjjjtzRLHvA',
2361                 'ext': 'mp4',
2362                 'title': 'ハッシュタグ無し };if window.ytcsi',
2363                 'upload_date': '20220323',
2364                 'like_count': int,
2365                 'availability': 'unlisted',
2366                 'channel': 'nao20010128nao',
2367                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2368                 'age_limit': 0,
2369                 'uploader': 'nao20010128nao',
2370                 'uploader_id': 'nao20010128nao',
2371                 'categories': ['Music'],
2372                 'view_count': int,
2373                 'description': '',
2374                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2375                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2376                 'live_status': 'not_live',
2377                 'playable_in_embed': True,
2378                 'channel_follower_count': int,
2379                 'duration': 6,
2380                 'tags': [],
2381                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2382             }
2383         }, {
2384             # Prefer primary title+description language metadata by default
2385             # Do not prefer translated description if primary is empty
2386             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2387             'info_dict': {
2388                 'id': 'el3E4MbxRqQ',
2389                 'ext': 'mp4',
2390                 'title': 'dlp test video 2 - primary sv no desc',
2391                 'description': '',
2392                 'channel': 'cole-dlp-test-acc',
2393                 'tags': [],
2394                 'view_count': int,
2395                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2396                 'like_count': int,
2397                 'playable_in_embed': True,
2398                 'availability': 'unlisted',
2399                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2400                 'age_limit': 0,
2401                 'duration': 5,
2402                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2403                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2404                 'live_status': 'not_live',
2405                 'upload_date': '20220908',
2406                 'categories': ['People & Blogs'],
2407                 'uploader': 'cole-dlp-test-acc',
2408                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2409             },
2410             'params': {'skip_download': True}
2411         }, {
2412             # Extractor argument: prefer translated title+description
2413             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2414             'info_dict': {
2415                 'id': 'gHKT4uU8Zng',
2416                 'ext': 'mp4',
2417                 'channel': 'cole-dlp-test-acc',
2418                 'tags': [],
2419                 'duration': 5,
2420                 'live_status': 'not_live',
2421                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2422                 'upload_date': '20220728',
2423                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2424                 'view_count': int,
2425                 'categories': ['People & Blogs'],
2426                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2427                 'title': 'dlp test video title translated (fr)',
2428                 'availability': 'public',
2429                 'uploader': 'cole-dlp-test-acc',
2430                 'age_limit': 0,
2431                 'description': 'dlp test video description translated (fr)',
2432                 'playable_in_embed': True,
2433                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2434                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2435             },
2436             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2437             'expected_warnings': [r'Preferring "fr" translated fields'],
2438         }, {
2439             'note': '6 channel audio',
2440             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2441             'only_matching': True,
2442         }
2443     ]
2444
2445     _WEBPAGE_TESTS = [
2446         # YouTube <object> embed
2447         {
2448             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2449             'md5': '873c81d308b979f0e23ee7e620b312a3',
2450             'info_dict': {
2451                 'id': 'msN87y-iEx0',
2452                 'ext': 'mp4',
2453                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2454                 'upload_date': '20080526',
2455                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2456                 'uploader': 'Christopher Sykes',
2457                 'uploader_id': 'ChristopherJSykes',
2458                 'age_limit': 0,
2459                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2460                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2461                 'playable_in_embed': True,
2462                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2463                 'like_count': int,
2464                 'comment_count': int,
2465                 'channel': 'Christopher Sykes',
2466                 'live_status': 'not_live',
2467                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2468                 'availability': 'public',
2469                 'duration': 195,
2470                 'view_count': int,
2471                 'categories': ['Science & Technology'],
2472                 'channel_follower_count': int,
2473                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2474             },
2475             'params': {
2476                 'skip_download': True,
2477             }
2478         },
2479     ]
2480
2481     @classmethod
2482     def suitable(cls, url):
2483         from ..utils import parse_qs
2484
2485         qs = parse_qs(url)
2486         if qs.get('list', [None])[0]:
2487             return False
2488         return super().suitable(url)
2489
2490     def __init__(self, *args, **kwargs):
2491         super().__init__(*args, **kwargs)
2492         self._code_cache = {}
2493         self._player_cache = {}
2494
2495     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2496         lock = threading.Lock()
2497
2498         is_live = True
2499         start_time = time.time()
2500         formats = [f for f in formats if f.get('is_from_start')]
2501
2502         def refetch_manifest(format_id, delay):
2503             nonlocal formats, start_time, is_live
2504             if time.time() <= start_time + delay:
2505                 return
2506
2507             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2508             video_details = traverse_obj(
2509                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2510             microformats = traverse_obj(
2511                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2512                 expected_type=dict, default=[])
2513             _, is_live, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2514             start_time = time.time()
2515
2516         def mpd_feed(format_id, delay):
2517             """
2518             @returns (manifest_url, manifest_stream_number, is_live) or None
2519             """
2520             with lock:
2521                 refetch_manifest(format_id, delay)
2522
2523             f = next((f for f in formats if f['format_id'] == format_id), None)
2524             if not f:
2525                 if not is_live:
2526                     self.to_screen(f'{video_id}: Video is no longer live')
2527                 else:
2528                     self.report_warning(
2529                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2530                 return None
2531             return f['manifest_url'], f['manifest_stream_number'], is_live
2532
2533         for f in formats:
2534             f['is_live'] = True
2535             f['protocol'] = 'http_dash_segments_generator'
2536             f['fragments'] = functools.partial(
2537                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2538
2539     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2540         FETCH_SPAN, MAX_DURATION = 5, 432000
2541
2542         mpd_url, stream_number, is_live = None, None, True
2543
2544         begin_index = 0
2545         download_start_time = ctx.get('start') or time.time()
2546
2547         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2548         if lack_early_segments:
2549             self.report_warning(bug_reports_message(
2550                 'Starting download from the last 120 hours of the live stream since '
2551                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2552             lack_early_segments = True
2553
2554         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2555         fragments, fragment_base_url = None, None
2556
2557         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2558             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2559             # Obtain from MPD's maximum seq value
2560             old_mpd_url = mpd_url
2561             last_error = ctx.pop('last_error', None)
2562             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2563             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2564                                                or (mpd_url, stream_number, False))
2565             if not refresh_sequence:
2566                 if expire_fast and not is_live:
2567                     return False, last_seq
2568                 elif old_mpd_url == mpd_url:
2569                     return True, last_seq
2570             try:
2571                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2572                     mpd_url, None, note=False, errnote=False, fatal=False)
2573             except ExtractorError:
2574                 fmts = None
2575             if not fmts:
2576                 no_fragment_score += 2
2577                 return False, last_seq
2578             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2579             fragments = fmt_info['fragments']
2580             fragment_base_url = fmt_info['fragment_base_url']
2581             assert fragment_base_url
2582
2583             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2584             return True, _last_seq
2585
2586         while is_live:
2587             fetch_time = time.time()
2588             if no_fragment_score > 30:
2589                 return
2590             if last_segment_url:
2591                 # Obtain from "X-Head-Seqnum" header value from each segment
2592                 try:
2593                     urlh = self._request_webpage(
2594                         last_segment_url, None, note=False, errnote=False, fatal=False)
2595                 except ExtractorError:
2596                     urlh = None
2597                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2598                 if last_seq is None:
2599                     no_fragment_score += 2
2600                     last_segment_url = None
2601                     continue
2602             else:
2603                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2604                 no_fragment_score += 2
2605                 if not should_continue:
2606                     continue
2607
2608             if known_idx > last_seq:
2609                 last_segment_url = None
2610                 continue
2611
2612             last_seq += 1
2613
2614             if begin_index < 0 and known_idx < 0:
2615                 # skip from the start when it's negative value
2616                 known_idx = last_seq + begin_index
2617             if lack_early_segments:
2618                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2619             try:
2620                 for idx in range(known_idx, last_seq):
2621                     # do not update sequence here or you'll get skipped some part of it
2622                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2623                     if not should_continue:
2624                         known_idx = idx - 1
2625                         raise ExtractorError('breaking out of outer loop')
2626                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2627                     yield {
2628                         'url': last_segment_url,
2629                         'fragment_count': last_seq,
2630                     }
2631                 if known_idx == last_seq:
2632                     no_fragment_score += 5
2633                 else:
2634                     no_fragment_score = 0
2635                 known_idx = last_seq
2636             except ExtractorError:
2637                 continue
2638
2639             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2640
2641     def _extract_player_url(self, *ytcfgs, webpage=None):
2642         player_url = traverse_obj(
2643             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2644             get_all=False, expected_type=str)
2645         if not player_url:
2646             return
2647         return urljoin('https://www.youtube.com', player_url)
2648
2649     def _download_player_url(self, video_id, fatal=False):
2650         res = self._download_webpage(
2651             'https://www.youtube.com/iframe_api',
2652             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2653         if res:
2654             player_version = self._search_regex(
2655                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2656             if player_version:
2657                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2658
2659     def _signature_cache_id(self, example_sig):
2660         """ Return a string representation of a signature """
2661         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2662
2663     @classmethod
2664     def _extract_player_info(cls, player_url):
2665         for player_re in cls._PLAYER_INFO_RE:
2666             id_m = re.search(player_re, player_url)
2667             if id_m:
2668                 break
2669         else:
2670             raise ExtractorError('Cannot identify player %r' % player_url)
2671         return id_m.group('id')
2672
2673     def _load_player(self, video_id, player_url, fatal=True):
2674         player_id = self._extract_player_info(player_url)
2675         if player_id not in self._code_cache:
2676             code = self._download_webpage(
2677                 player_url, video_id, fatal=fatal,
2678                 note='Downloading player ' + player_id,
2679                 errnote='Download of %s failed' % player_url)
2680             if code:
2681                 self._code_cache[player_id] = code
2682         return self._code_cache.get(player_id)
2683
2684     def _extract_signature_function(self, video_id, player_url, example_sig):
2685         player_id = self._extract_player_info(player_url)
2686
2687         # Read from filesystem cache
2688         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2689         assert os.path.basename(func_id) == func_id
2690
2691         self.write_debug(f'Extracting signature function {func_id}')
2692         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2693
2694         if not cache_spec:
2695             code = self._load_player(video_id, player_url)
2696         if code:
2697             res = self._parse_sig_js(code)
2698             test_string = ''.join(map(chr, range(len(example_sig))))
2699             cache_spec = [ord(c) for c in res(test_string)]
2700             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2701
2702         return lambda s: ''.join(s[i] for i in cache_spec)
2703
2704     def _print_sig_code(self, func, example_sig):
2705         if not self.get_param('youtube_print_sig_code'):
2706             return
2707
2708         def gen_sig_code(idxs):
2709             def _genslice(start, end, step):
2710                 starts = '' if start == 0 else str(start)
2711                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2712                 steps = '' if step == 1 else (':%d' % step)
2713                 return f's[{starts}{ends}{steps}]'
2714
2715             step = None
2716             # Quelch pyflakes warnings - start will be set when step is set
2717             start = '(Never used)'
2718             for i, prev in zip(idxs[1:], idxs[:-1]):
2719                 if step is not None:
2720                     if i - prev == step:
2721                         continue
2722                     yield _genslice(start, prev, step)
2723                     step = None
2724                     continue
2725                 if i - prev in [-1, 1]:
2726                     step = i - prev
2727                     start = prev
2728                     continue
2729                 else:
2730                     yield 's[%d]' % prev
2731             if step is None:
2732                 yield 's[%d]' % i
2733             else:
2734                 yield _genslice(start, i, step)
2735
2736         test_string = ''.join(map(chr, range(len(example_sig))))
2737         cache_res = func(test_string)
2738         cache_spec = [ord(c) for c in cache_res]
2739         expr_code = ' + '.join(gen_sig_code(cache_spec))
2740         signature_id_tuple = '(%s)' % (
2741             ', '.join(str(len(p)) for p in example_sig.split('.')))
2742         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2743                 '    return %s\n') % (signature_id_tuple, expr_code)
2744         self.to_screen('Extracted signature function:\n' + code)
2745
2746     def _parse_sig_js(self, jscode):
2747         funcname = self._search_regex(
2748             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2749              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2750              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2751              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2752              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2753              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2754              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2755              # Obsolete patterns
2756              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2757              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2758              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2759              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2760              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2761              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2762              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2763              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2764             jscode, 'Initial JS player signature function name', group='sig')
2765
2766         jsi = JSInterpreter(jscode)
2767         initial_function = jsi.extract_function(funcname)
2768         return lambda s: initial_function([s])
2769
2770     def _cached(self, func, *cache_id):
2771         def inner(*args, **kwargs):
2772             if cache_id not in self._player_cache:
2773                 try:
2774                     self._player_cache[cache_id] = func(*args, **kwargs)
2775                 except ExtractorError as e:
2776                     self._player_cache[cache_id] = e
2777                 except Exception as e:
2778                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
2779
2780             ret = self._player_cache[cache_id]
2781             if isinstance(ret, Exception):
2782                 raise ret
2783             return ret
2784         return inner
2785
2786     def _decrypt_signature(self, s, video_id, player_url):
2787         """Turn the encrypted s field into a working signature"""
2788         extract_sig = self._cached(
2789             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
2790         func = extract_sig(video_id, player_url, s)
2791         self._print_sig_code(func, s)
2792         return func(s)
2793
2794     def _decrypt_nsig(self, s, video_id, player_url):
2795         """Turn the encrypted n field into a working signature"""
2796         if player_url is None:
2797             raise ExtractorError('Cannot decrypt nsig without player_url')
2798         player_url = urljoin('https://www.youtube.com', player_url)
2799
2800         try:
2801             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
2802         except ExtractorError as e:
2803             raise ExtractorError('Unable to extract nsig function code', cause=e)
2804         if self.get_param('youtube_print_sig_code'):
2805             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2806
2807         try:
2808             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
2809             ret = extract_nsig(jsi, func_code)(s)
2810         except JSInterpreter.Exception as e:
2811             try:
2812                 jsi = PhantomJSwrapper(self, timeout=5000)
2813             except ExtractorError:
2814                 raise e
2815             self.report_warning(
2816                 f'Native nsig extraction failed: Trying with PhantomJS\n'
2817                 f'         n = {s} ; player = {player_url}', video_id)
2818             self.write_debug(e)
2819
2820             args, func_body = func_code
2821             ret = jsi.execute(
2822                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
2823                 video_id=video_id, note='Executing signature code').strip()
2824
2825         self.write_debug(f'Decrypted nsig {s} => {ret}')
2826         return ret
2827
2828     def _extract_n_function_name(self, jscode):
2829         funcname, idx = self._search_regex(
2830             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2831             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2832         if not idx:
2833             return funcname
2834
2835         return json.loads(js_to_json(self._search_regex(
2836             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
2837             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
2838
2839     def _extract_n_function_code(self, video_id, player_url):
2840         player_id = self._extract_player_info(player_url)
2841         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
2842         jscode = func_code or self._load_player(video_id, player_url)
2843         jsi = JSInterpreter(jscode)
2844
2845         if func_code:
2846             return jsi, player_id, func_code
2847
2848         func_name = self._extract_n_function_name(jscode)
2849
2850         # For redundancy
2851         func_code = self._search_regex(
2852             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
2853                      # NB: The end of the regex is intentionally kept strict
2854                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
2855             jscode, 'nsig function', group=('var', 'code'), default=None)
2856         if func_code:
2857             func_code = ([func_code[0]], func_code[1])
2858         else:
2859             self.write_debug('Extracting nsig function with jsinterp')
2860             func_code = jsi.extract_function_code(func_name)
2861
2862         self.cache.store('youtube-nsig', player_id, func_code)
2863         return jsi, player_id, func_code
2864
2865     def _extract_n_function_from_code(self, jsi, func_code):
2866         func = jsi.extract_function_from_code(*func_code)
2867
2868         def extract_nsig(s):
2869             try:
2870                 ret = func([s])
2871             except JSInterpreter.Exception:
2872                 raise
2873             except Exception as e:
2874                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
2875
2876             if ret.startswith('enhanced_except_'):
2877                 raise JSInterpreter.Exception('Signature function returned an exception')
2878             return ret
2879
2880         return extract_nsig
2881
2882     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2883         """
2884         Extract signatureTimestamp (sts)
2885         Required to tell API what sig/player version is in use.
2886         """
2887         sts = None
2888         if isinstance(ytcfg, dict):
2889             sts = int_or_none(ytcfg.get('STS'))
2890
2891         if not sts:
2892             # Attempt to extract from player
2893             if player_url is None:
2894                 error_msg = 'Cannot extract signature timestamp without player_url.'
2895                 if fatal:
2896                     raise ExtractorError(error_msg)
2897                 self.report_warning(error_msg)
2898                 return
2899             code = self._load_player(video_id, player_url, fatal=fatal)
2900             if code:
2901                 sts = int_or_none(self._search_regex(
2902                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2903                     'JS player signature timestamp', group='sts', fatal=fatal))
2904         return sts
2905
2906     def _mark_watched(self, video_id, player_responses):
2907         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2908             label = 'fully ' if is_full else ''
2909             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2910                             expected_type=url_or_none)
2911             if not url:
2912                 self.report_warning(f'Unable to mark {label}watched')
2913                 return
2914             parsed_url = urllib.parse.urlparse(url)
2915             qs = urllib.parse.parse_qs(parsed_url.query)
2916
2917             # cpn generation algorithm is reverse engineered from base.js.
2918             # In fact it works even with dummy cpn.
2919             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2920             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2921
2922             # # more consistent results setting it to right before the end
2923             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2924
2925             qs.update({
2926                 'ver': ['2'],
2927                 'cpn': [cpn],
2928                 'cmt': video_length,
2929                 'el': 'detailpage',  # otherwise defaults to "shorts"
2930             })
2931
2932             if is_full:
2933                 # these seem to mark watchtime "history" in the real world
2934                 # they're required, so send in a single value
2935                 qs.update({
2936                     'st': video_length,
2937                     'et': video_length,
2938                 })
2939
2940             url = urllib.parse.urlunparse(
2941                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2942
2943             self._download_webpage(
2944                 url, video_id, f'Marking {label}watched',
2945                 'Unable to mark watched', fatal=False)
2946
2947     @classmethod
2948     def _extract_from_webpage(cls, url, webpage):
2949         # Invidious Instances
2950         # https://github.com/yt-dlp/yt-dlp/issues/195
2951         # https://github.com/iv-org/invidious/pull/1730
2952         mobj = re.search(
2953             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
2954             webpage)
2955         if mobj:
2956             yield cls.url_result(mobj.group('url'), cls)
2957             raise cls.StopExtraction()
2958
2959         yield from super()._extract_from_webpage(url, webpage)
2960
2961         # lazyYT YouTube embed
2962         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
2963             yield cls.url_result(unescapeHTML(id_), cls, id_)
2964
2965         # Wordpress "YouTube Video Importer" plugin
2966         for m in re.findall(r'''(?x)<div[^>]+
2967                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2968                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
2969             yield cls.url_result(m[-1], cls, m[-1])
2970
2971     @classmethod
2972     def extract_id(cls, url):
2973         video_id = cls.get_temp_id(url)
2974         if not video_id:
2975             raise ExtractorError(f'Invalid URL: {url}')
2976         return video_id
2977
2978     def _extract_chapters_from_json(self, data, duration):
2979         chapter_list = traverse_obj(
2980             data, (
2981                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2982                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2983             ), expected_type=list)
2984
2985         return self._extract_chapters(
2986             chapter_list,
2987             chapter_time=lambda chapter: float_or_none(
2988                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2989             chapter_title=lambda chapter: traverse_obj(
2990                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2991             duration=duration)
2992
2993     def _extract_chapters_from_engagement_panel(self, data, duration):
2994         content_list = traverse_obj(
2995             data,
2996             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2997             expected_type=list, default=[])
2998         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2999         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3000
3001         return next(filter(None, (
3002             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3003                                    chapter_time, chapter_title, duration)
3004             for contents in content_list)), [])
3005
3006     def _extract_chapters_from_description(self, description, duration):
3007         return self._extract_chapters(
3008             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
3009             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
3010             duration=duration, strict=False)
3011
3012     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
3013         if not duration:
3014             return
3015         chapter_list = [{
3016             'start_time': chapter_time(chapter),
3017             'title': chapter_title(chapter),
3018         } for chapter in chapter_list or []]
3019         if not strict:
3020             chapter_list.sort(key=lambda c: c['start_time'] or 0)
3021
3022         chapters = [{'start_time': 0}]
3023         for idx, chapter in enumerate(chapter_list):
3024             if chapter['start_time'] is None:
3025                 self.report_warning(f'Incomplete chapter {idx}')
3026             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
3027                 chapters.append(chapter)
3028             elif chapter not in chapters:
3029                 self.report_warning(
3030                     f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
3031         return chapters[1:]
3032
3033     def _extract_comment(self, comment_renderer, parent=None):
3034         comment_id = comment_renderer.get('commentId')
3035         if not comment_id:
3036             return
3037
3038         text = self._get_text(comment_renderer, 'contentText')
3039
3040         # Timestamp is an estimate calculated from the current time and time_text
3041         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3042         timestamp = self._parse_time_text(time_text)
3043
3044         author = self._get_text(comment_renderer, 'authorText')
3045         author_id = try_get(comment_renderer,
3046                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3047
3048         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3049                                                        lambda x: x['likeCount']), str)) or 0
3050         author_thumbnail = try_get(comment_renderer,
3051                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3052
3053         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3054         is_favorited = 'creatorHeart' in (try_get(
3055             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3056         return {
3057             'id': comment_id,
3058             'text': text,
3059             'timestamp': timestamp,
3060             'time_text': time_text,
3061             'like_count': votes,
3062             'is_favorited': is_favorited,
3063             'author': author,
3064             'author_id': author_id,
3065             'author_thumbnail': author_thumbnail,
3066             'author_is_uploader': author_is_uploader,
3067             'parent': parent or 'root'
3068         }
3069
3070     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3071
3072         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3073
3074         def extract_header(contents):
3075             _continuation = None
3076             for content in contents:
3077                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3078                 expected_comment_count = self._get_count(
3079                     comments_header_renderer, 'countText', 'commentsCount')
3080
3081                 if expected_comment_count:
3082                     tracker['est_total'] = expected_comment_count
3083                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3084                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3085
3086                 sort_menu_item = try_get(
3087                     comments_header_renderer,
3088                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3089                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3090
3091                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3092                 if not _continuation:
3093                     continue
3094
3095                 sort_text = str_or_none(sort_menu_item.get('title'))
3096                 if not sort_text:
3097                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3098                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3099                 break
3100             return _continuation
3101
3102         def extract_thread(contents):
3103             if not parent:
3104                 tracker['current_page_thread'] = 0
3105             for content in contents:
3106                 if not parent and tracker['total_parent_comments'] >= max_parents:
3107                     yield
3108                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3109                 comment_renderer = get_first(
3110                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3111                     expected_type=dict, default={})
3112
3113                 comment = self._extract_comment(comment_renderer, parent)
3114                 if not comment:
3115                     continue
3116
3117                 tracker['running_total'] += 1
3118                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3119                 yield comment
3120
3121                 # Attempt to get the replies
3122                 comment_replies_renderer = try_get(
3123                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3124
3125                 if comment_replies_renderer:
3126                     tracker['current_page_thread'] += 1
3127                     comment_entries_iter = self._comment_entries(
3128                         comment_replies_renderer, ytcfg, video_id,
3129                         parent=comment.get('id'), tracker=tracker)
3130                     yield from itertools.islice(comment_entries_iter, min(
3131                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3132
3133         # Keeps track of counts across recursive calls
3134         if not tracker:
3135             tracker = dict(
3136                 running_total=0,
3137                 est_total=0,
3138                 current_page_thread=0,
3139                 total_parent_comments=0,
3140                 total_reply_comments=0)
3141
3142         # TODO: Deprecated
3143         # YouTube comments have a max depth of 2
3144         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3145         if max_depth:
3146             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3147                                                 'Set max replies in the max-comments extractor argument instead')
3148         if max_depth == 1 and parent:
3149             return
3150
3151         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3152             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3153
3154         continuation = self._extract_continuation(root_continuation_data)
3155
3156         response = None
3157         is_forced_continuation = False
3158         is_first_continuation = parent is None
3159         if is_first_continuation and not continuation:
3160             # Sometimes you can get comments by generating the continuation yourself,
3161             # even if YouTube initially reports them being disabled - e.g. stories comments.
3162             # Note: if the comment section is actually disabled, YouTube may return a response with
3163             # required check_get_keys missing. So we will disable that check initially in this case.
3164             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3165             is_forced_continuation = True
3166
3167         for page_num in itertools.count(0):
3168             if not continuation:
3169                 break
3170             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3171             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3172             if page_num == 0:
3173                 if is_first_continuation:
3174                     note_prefix = 'Downloading comment section API JSON'
3175                 else:
3176                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3177                         tracker['current_page_thread'], comment_prog_str)
3178             else:
3179                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3180                     '       ' if parent else '', ' replies' if parent else '',
3181                     page_num, comment_prog_str)
3182
3183             response = self._extract_response(
3184                 item_id=None, query=continuation,
3185                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3186                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3187             is_forced_continuation = False
3188             continuation_contents = traverse_obj(
3189                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3190
3191             continuation = None
3192             for continuation_section in continuation_contents:
3193                 continuation_items = traverse_obj(
3194                     continuation_section,
3195                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3196                     get_all=False, expected_type=list) or []
3197                 if is_first_continuation:
3198                     continuation = extract_header(continuation_items)
3199                     is_first_continuation = False
3200                     if continuation:
3201                         break
3202                     continue
3203
3204                 for entry in extract_thread(continuation_items):
3205                     if not entry:
3206                         return
3207                     yield entry
3208                 continuation = self._extract_continuation({'contents': continuation_items})
3209                 if continuation:
3210                     break
3211
3212         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3213         if message and not parent and tracker['running_total'] == 0:
3214             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3215
3216     @staticmethod
3217     def _generate_comment_continuation(video_id):
3218         """
3219         Generates initial comment section continuation token from given video id
3220         """
3221         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3222         return base64.b64encode(token.encode()).decode()
3223
3224     def _get_comments(self, ytcfg, video_id, contents, webpage):
3225         """Entry for comment extraction"""
3226         def _real_comment_extract(contents):
3227             renderer = next((
3228                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3229                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3230             yield from self._comment_entries(renderer, ytcfg, video_id)
3231
3232         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3233         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3234
3235     @staticmethod
3236     def _get_checkok_params():
3237         return {'contentCheckOk': True, 'racyCheckOk': True}
3238
3239     @classmethod
3240     def _generate_player_context(cls, sts=None):
3241         context = {
3242             'html5Preference': 'HTML5_PREF_WANTS',
3243         }
3244         if sts is not None:
3245             context['signatureTimestamp'] = sts
3246         return {
3247             'playbackContext': {
3248                 'contentPlaybackContext': context
3249             },
3250             **cls._get_checkok_params()
3251         }
3252
3253     @staticmethod
3254     def _is_agegated(player_response):
3255         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3256             return True
3257
3258         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3259         AGE_GATE_REASONS = (
3260             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3261             'age_verification_required', 'age_check_required',  # status
3262         )
3263         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3264
3265     @staticmethod
3266     def _is_unplayable(player_response):
3267         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3268
3269     _STORY_PLAYER_PARAMS = '8AEB'
3270
3271     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3272
3273         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3274         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3275         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3276         headers = self.generate_api_headers(
3277             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3278
3279         yt_query = {
3280             'videoId': video_id,
3281         }
3282         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3283             yt_query['params'] = self._STORY_PLAYER_PARAMS
3284
3285         yt_query.update(self._generate_player_context(sts))
3286         return self._extract_response(
3287             item_id=video_id, ep='player', query=yt_query,
3288             ytcfg=player_ytcfg, headers=headers, fatal=True,
3289             default_client=client,
3290             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3291         ) or None
3292
3293     def _get_requested_clients(self, url, smuggled_data):
3294         requested_clients = []
3295         default = ['android', 'web']
3296         allowed_clients = sorted(
3297             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3298             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3299         for client in self._configuration_arg('player_client'):
3300             if client in allowed_clients:
3301                 requested_clients.append(client)
3302             elif client == 'default':
3303                 requested_clients.extend(default)
3304             elif client == 'all':
3305                 requested_clients.extend(allowed_clients)
3306             else:
3307                 self.report_warning(f'Skipping unsupported client {client}')
3308         if not requested_clients:
3309             requested_clients = default
3310
3311         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3312             requested_clients.extend(
3313                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3314
3315         return orderedSet(requested_clients)
3316
3317     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3318         initial_pr = None
3319         if webpage:
3320             initial_pr = self._search_json(
3321                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3322
3323         all_clients = set(clients)
3324         clients = clients[::-1]
3325         prs = []
3326
3327         def append_client(*client_names):
3328             """ Append the first client name that exists but not already used """
3329             for client_name in client_names:
3330                 actual_client = _split_innertube_client(client_name)[0]
3331                 if actual_client in INNERTUBE_CLIENTS:
3332                     if actual_client not in all_clients:
3333                         clients.append(client_name)
3334                         all_clients.add(actual_client)
3335                         return
3336
3337         # Android player_response does not have microFormats which are needed for
3338         # extraction of some data. So we return the initial_pr with formats
3339         # stripped out even if not requested by the user
3340         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3341         if initial_pr:
3342             pr = dict(initial_pr)
3343             pr['streamingData'] = None
3344             prs.append(pr)
3345
3346         last_error = None
3347         tried_iframe_fallback = False
3348         player_url = None
3349         while clients:
3350             client, base_client, variant = _split_innertube_client(clients.pop())
3351             player_ytcfg = master_ytcfg if client == 'web' else {}
3352             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3353                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3354
3355             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3356             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3357             if 'js' in self._configuration_arg('player_skip'):
3358                 require_js_player = False
3359                 player_url = None
3360
3361             if not player_url and not tried_iframe_fallback and require_js_player:
3362                 player_url = self._download_player_url(video_id)
3363                 tried_iframe_fallback = True
3364
3365             try:
3366                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3367                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3368             except ExtractorError as e:
3369                 if last_error:
3370                     self.report_warning(last_error)
3371                 last_error = e
3372                 continue
3373
3374             if pr:
3375                 # YouTube may return a different video player response than expected.
3376                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3377                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3378                 if pr_video_id and pr_video_id != video_id:
3379                     self.report_warning(
3380                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3381                 else:
3382                     prs.append(pr)
3383
3384             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3385             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3386                 append_client(f'{base_client}_creator')
3387             elif self._is_agegated(pr):
3388                 if variant == 'tv_embedded':
3389                     append_client(f'{base_client}_embedded')
3390                 elif not variant:
3391                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3392
3393         if last_error:
3394             if not len(prs):
3395                 raise last_error
3396             self.report_warning(last_error)
3397         return prs, player_url
3398
3399     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, is_live, duration):
3400         itags, stream_ids = {}, []
3401         itag_qualities, res_qualities = {}, {0: None}
3402         q = qualities([
3403             # Normally tiny is the smallest video-only formats. But
3404             # audio-only formats with unknown quality may get tagged as tiny
3405             'tiny',
3406             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3407             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3408         ])
3409         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3410
3411         for fmt in streaming_formats:
3412             if fmt.get('targetDurationSec'):
3413                 continue
3414
3415             itag = str_or_none(fmt.get('itag'))
3416             audio_track = fmt.get('audioTrack') or {}
3417             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3418             if stream_id in stream_ids:
3419                 continue
3420
3421             quality = fmt.get('quality')
3422             height = int_or_none(fmt.get('height'))
3423             if quality == 'tiny' or not quality:
3424                 quality = fmt.get('audioQuality', '').lower() or quality
3425             # The 3gp format (17) in android client has a quality of "small",
3426             # but is actually worse than other formats
3427             if itag == '17':
3428                 quality = 'tiny'
3429             if quality:
3430                 if itag:
3431                     itag_qualities[itag] = quality
3432                 if height:
3433                     res_qualities[height] = quality
3434             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3435             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3436             # number of fragment that would subsequently requested with (`&sq=N`)
3437             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3438                 continue
3439
3440             fmt_url = fmt.get('url')
3441             if not fmt_url:
3442                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3443                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3444                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3445                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3446                     continue
3447                 try:
3448                     fmt_url += '&%s=%s' % (
3449                         traverse_obj(sc, ('sp', -1)) or 'signature',
3450                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3451                     )
3452                 except ExtractorError as e:
3453                     self.report_warning('Signature extraction failed: Some formats may be missing',
3454                                         video_id=video_id, only_once=True)
3455                     self.write_debug(e, only_once=True)
3456                     continue
3457
3458             query = parse_qs(fmt_url)
3459             throttled = False
3460             if query.get('n'):
3461                 try:
3462                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3463                     fmt_url = update_url_query(fmt_url, {
3464                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3465                     })
3466                 except ExtractorError as e:
3467                     phantomjs_hint = ''
3468                     if isinstance(e, JSInterpreter.Exception):
3469                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3470                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3471                     if player_url:
3472                         self.report_warning(
3473                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3474                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3475                         self.write_debug(e, only_once=True)
3476                     else:
3477                         self.report_warning(
3478                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3479                             video_id=video_id, only_once=True)
3480                     throttled = True
3481
3482             if itag:
3483                 itags[itag] = 'https'
3484                 stream_ids.append(stream_id)
3485
3486             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3487             language_preference = (
3488                 10 if audio_track.get('audioIsDefault') and 10
3489                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3490                 else -1)
3491             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3492             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3493             # Make sure to avoid false positives with small duration differences.
3494             # E.g. __2ABJjxzNo, ySuUZEjARPY
3495             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3496             if is_damaged:
3497                 self.report_warning(
3498                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3499             dct = {
3500                 'asr': int_or_none(fmt.get('audioSampleRate')),
3501                 'filesize': int_or_none(fmt.get('contentLength')),
3502                 'format_id': itag,
3503                 'format_note': join_nonempty(
3504                     '%s%s' % (audio_track.get('displayName') or '',
3505                               ' (default)' if language_preference > 0 else ''),
3506                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3507                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3508                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3509                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3510                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3511                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3512                 'fps': int_or_none(fmt.get('fps')) or None,
3513                 'audio_channels': fmt.get('audioChannels'),
3514                 'height': height,
3515                 'quality': q(quality),
3516                 'has_drm': bool(fmt.get('drmFamilies')),
3517                 'tbr': tbr,
3518                 'url': fmt_url,
3519                 'width': int_or_none(fmt.get('width')),
3520                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3521                                           'desc' if language_preference < -1 else ''),
3522                 'language_preference': language_preference,
3523                 # Strictly de-prioritize damaged and 3gp formats
3524                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3525             }
3526             mime_mobj = re.match(
3527                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3528             if mime_mobj:
3529                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3530                 dct.update(parse_codecs(mime_mobj.group(2)))
3531             no_audio = dct.get('acodec') == 'none'
3532             no_video = dct.get('vcodec') == 'none'
3533             if no_audio:
3534                 dct['vbr'] = tbr
3535             if no_video:
3536                 dct['abr'] = tbr
3537             if no_audio or no_video:
3538                 dct['downloader_options'] = {
3539                     # Youtube throttles chunks >~10M
3540                     'http_chunk_size': 10485760,
3541                 }
3542                 if dct.get('ext'):
3543                     dct['container'] = dct['ext'] + '_dash'
3544             yield dct
3545
3546         live_from_start = is_live and self.get_param('live_from_start')
3547         skip_manifests = self._configuration_arg('skip')
3548         if not self.get_param('youtube_include_hls_manifest', True):
3549             skip_manifests.append('hls')
3550         if not self.get_param('youtube_include_dash_manifest', True):
3551             skip_manifests.append('dash')
3552         get_dash = 'dash' not in skip_manifests and (
3553             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3554         get_hls = not live_from_start and 'hls' not in skip_manifests
3555
3556         def process_manifest_format(f, proto, itag):
3557             if itag in itags:
3558                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3559                     return False
3560                 itag = f'{itag}-{proto}'
3561             if itag:
3562                 f['format_id'] = itag
3563                 itags[itag] = proto
3564
3565             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3566             if f['quality'] == -1 and f.get('height'):
3567                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3568             return True
3569
3570         subtitles = {}
3571         for sd in streaming_data:
3572             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3573             if hls_manifest_url:
3574                 fmts, subs = self._extract_m3u8_formats_and_subtitles(hls_manifest_url, video_id, 'mp4', fatal=False, live=is_live)
3575                 subtitles = self._merge_subtitles(subs, subtitles)
3576                 for f in fmts:
3577                     if process_manifest_format(f, 'hls', self._search_regex(
3578                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3579                         yield f
3580
3581             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3582             if dash_manifest_url:
3583                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3584                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3585                 for f in formats:
3586                     if process_manifest_format(f, 'dash', f['format_id']):
3587                         f['filesize'] = int_or_none(self._search_regex(
3588                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3589                         if live_from_start:
3590                             f['is_from_start'] = True
3591
3592                         yield f
3593         yield subtitles
3594
3595     def _extract_storyboard(self, player_responses, duration):
3596         spec = get_first(
3597             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3598         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3599         if not base_url:
3600             return
3601         L = len(spec) - 1
3602         for i, args in enumerate(spec):
3603             args = args.split('#')
3604             counts = list(map(int_or_none, args[:5]))
3605             if len(args) != 8 or not all(counts):
3606                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3607                 continue
3608             width, height, frame_count, cols, rows = counts
3609             N, sigh = args[6:]
3610
3611             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3612             fragment_count = frame_count / (cols * rows)
3613             fragment_duration = duration / fragment_count
3614             yield {
3615                 'format_id': f'sb{i}',
3616                 'format_note': 'storyboard',
3617                 'ext': 'mhtml',
3618                 'protocol': 'mhtml',
3619                 'acodec': 'none',
3620                 'vcodec': 'none',
3621                 'url': url,
3622                 'width': width,
3623                 'height': height,
3624                 'fps': frame_count / duration,
3625                 'rows': rows,
3626                 'columns': cols,
3627                 'fragments': [{
3628                     'url': url.replace('$M', str(j)),
3629                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3630                 } for j in range(math.ceil(fragment_count))],
3631             }
3632
3633     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3634         webpage = None
3635         if 'webpage' not in self._configuration_arg('player_skip'):
3636             query = {'bpctr': '9999999999', 'has_verified': '1'}
3637             if smuggled_data.get('is_story'):
3638                 query['pp'] = self._STORY_PLAYER_PARAMS
3639             webpage = self._download_webpage(
3640                 webpage_url, video_id, fatal=False, query=query)
3641
3642         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3643
3644         player_responses, player_url = self._extract_player_responses(
3645             self._get_requested_clients(url, smuggled_data),
3646             video_id, webpage, master_ytcfg, smuggled_data)
3647
3648         return webpage, master_ytcfg, player_responses, player_url
3649
3650     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3651         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3652         is_live = get_first(video_details, 'isLive')
3653         if is_live is None:
3654             is_live = get_first(live_broadcast_details, 'isLiveNow')
3655
3656         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3657         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, is_live, duration)
3658
3659         return live_broadcast_details, is_live, streaming_data, formats, subtitles
3660
3661     def _real_extract(self, url):
3662         url, smuggled_data = unsmuggle_url(url, {})
3663         video_id = self._match_id(url)
3664
3665         base_url = self.http_scheme() + '//www.youtube.com/'
3666         webpage_url = base_url + 'watch?v=' + video_id
3667
3668         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3669
3670         playability_statuses = traverse_obj(
3671             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3672
3673         trailer_video_id = get_first(
3674             playability_statuses,
3675             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3676             expected_type=str)
3677         if trailer_video_id:
3678             return self.url_result(
3679                 trailer_video_id, self.ie_key(), trailer_video_id)
3680
3681         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3682                        if webpage else (lambda x: None))
3683
3684         video_details = traverse_obj(
3685             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3686         microformats = traverse_obj(
3687             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3688             expected_type=dict, default=[])
3689
3690         translated_title = self._get_text(microformats, (..., 'title'))
3691         video_title = (self._preferred_lang and translated_title
3692                        or get_first(video_details, 'title')  # primary
3693                        or translated_title
3694                        or search_meta(['og:title', 'twitter:title', 'title']))
3695         translated_description = self._get_text(microformats, (..., 'description'))
3696         original_description = get_first(video_details, 'shortDescription')
3697         video_description = (
3698             self._preferred_lang and translated_description
3699             # If original description is blank, it will be an empty string.
3700             # Do not prefer translated description in this case.
3701             or original_description if original_description is not None else translated_description)
3702
3703         multifeed_metadata_list = get_first(
3704             player_responses,
3705             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3706             expected_type=str)
3707         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3708             if self.get_param('noplaylist'):
3709                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3710             else:
3711                 entries = []
3712                 feed_ids = []
3713                 for feed in multifeed_metadata_list.split(','):
3714                     # Unquote should take place before split on comma (,) since textual
3715                     # fields may contain comma as well (see
3716                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3717                     feed_data = urllib.parse.parse_qs(
3718                         urllib.parse.unquote_plus(feed))
3719
3720                     def feed_entry(name):
3721                         return try_get(
3722                             feed_data, lambda x: x[name][0], str)
3723
3724                     feed_id = feed_entry('id')
3725                     if not feed_id:
3726                         continue
3727                     feed_title = feed_entry('title')
3728                     title = video_title
3729                     if feed_title:
3730                         title += ' (%s)' % feed_title
3731                     entries.append({
3732                         '_type': 'url_transparent',
3733                         'ie_key': 'Youtube',
3734                         'url': smuggle_url(
3735                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3736                             {'force_singlefeed': True}),
3737                         'title': title,
3738                     })
3739                     feed_ids.append(feed_id)
3740                 self.to_screen(
3741                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3742                     % (', '.join(feed_ids), video_id))
3743                 return self.playlist_result(
3744                     entries, video_id, video_title, video_description)
3745
3746         duration = int_or_none(
3747             get_first(video_details, 'lengthSeconds')
3748             or get_first(microformats, 'lengthSeconds')
3749             or parse_duration(search_meta('duration'))) or None
3750
3751         live_broadcast_details, is_live, streaming_data, formats, automatic_captions = \
3752             self._list_formats(video_id, microformats, video_details, player_responses, player_url)
3753
3754         if not formats:
3755             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3756                 self.report_drm(video_id)
3757             pemr = get_first(
3758                 playability_statuses,
3759                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3760             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3761             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3762             if subreason:
3763                 if subreason == 'The uploader has not made this video available in your country.':
3764                     countries = get_first(microformats, 'availableCountries')
3765                     if not countries:
3766                         regions_allowed = search_meta('regionsAllowed')
3767                         countries = regions_allowed.split(',') if regions_allowed else None
3768                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3769                 reason += f'. {subreason}'
3770             if reason:
3771                 self.raise_no_formats(reason, expected=True)
3772
3773         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3774         if not keywords and webpage:
3775             keywords = [
3776                 unescapeHTML(m.group('content'))
3777                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3778         for keyword in keywords:
3779             if keyword.startswith('yt:stretch='):
3780                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3781                 if mobj:
3782                     # NB: float is intentional for forcing float division
3783                     w, h = (float(v) for v in mobj.groups())
3784                     if w > 0 and h > 0:
3785                         ratio = w / h
3786                         for f in formats:
3787                             if f.get('vcodec') != 'none':
3788                                 f['stretched_ratio'] = ratio
3789                         break
3790         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3791         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3792         if thumbnail_url:
3793             thumbnails.append({
3794                 'url': thumbnail_url,
3795             })
3796         original_thumbnails = thumbnails.copy()
3797
3798         # The best resolution thumbnails sometimes does not appear in the webpage
3799         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3800         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3801         thumbnail_names = [
3802             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3803             # in resolution, these are not the custom thumbnail. So de-prioritize them
3804             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3805             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3806         ]
3807         n_thumbnail_names = len(thumbnail_names)
3808         thumbnails.extend({
3809             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3810                 video_id=video_id, name=name, ext=ext,
3811                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3812         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3813         for thumb in thumbnails:
3814             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3815             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3816         self._remove_duplicate_formats(thumbnails)
3817         self._downloader._sort_thumbnails(original_thumbnails)
3818
3819         category = get_first(microformats, 'category') or search_meta('genre')
3820         channel_id = str_or_none(
3821             get_first(video_details, 'channelId')
3822             or get_first(microformats, 'externalChannelId')
3823             or search_meta('channelId'))
3824         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3825
3826         live_content = get_first(video_details, 'isLiveContent')
3827         is_upcoming = get_first(video_details, 'isUpcoming')
3828         if is_live is None:
3829             if is_upcoming or live_content is False:
3830                 is_live = False
3831         if is_upcoming is None and (live_content or is_live):
3832             is_upcoming = False
3833         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3834         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3835         if not duration and live_end_time and live_start_time:
3836             duration = live_end_time - live_start_time
3837
3838         if is_live and self.get_param('live_from_start'):
3839             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3840
3841         formats.extend(self._extract_storyboard(player_responses, duration))
3842
3843         # source_preference is lower for throttled/potentially damaged formats
3844         self._sort_formats(formats, (
3845             'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto'))
3846
3847         info = {
3848             'id': video_id,
3849             'title': video_title,
3850             'formats': formats,
3851             'thumbnails': thumbnails,
3852             # The best thumbnail that we are sure exists. Prevents unnecessary
3853             # URL checking if user don't care about getting the best possible thumbnail
3854             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3855             'description': video_description,
3856             'uploader': get_first(video_details, 'author'),
3857             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3858             'uploader_url': owner_profile_url,
3859             'channel_id': channel_id,
3860             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3861             'duration': duration,
3862             'view_count': int_or_none(
3863                 get_first((video_details, microformats), (..., 'viewCount'))
3864                 or search_meta('interactionCount')),
3865             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3866             'age_limit': 18 if (
3867                 get_first(microformats, 'isFamilySafe') is False
3868                 or search_meta('isFamilyFriendly') == 'false'
3869                 or search_meta('og:restrictions:age') == '18+') else 0,
3870             'webpage_url': webpage_url,
3871             'categories': [category] if category else None,
3872             'tags': keywords,
3873             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3874             'is_live': is_live,
3875             'was_live': (False if is_live or is_upcoming or live_content is False
3876                          else None if is_live is None or is_upcoming is None
3877                          else live_content),
3878             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3879             'release_timestamp': live_start_time,
3880         }
3881
3882         if get_first(video_details, 'isPostLiveDvr'):
3883             self.write_debug('Video is in Post-Live Manifestless mode')
3884             info['live_status'] = 'post_live'
3885             if (duration or 0) > 4 * 3600:
3886                 self.report_warning(
3887                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3888                     'This is a known issue and patches are welcome')
3889
3890         subtitles = {}
3891         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3892         if pctr:
3893             def get_lang_code(track):
3894                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3895                         or track.get('languageCode'))
3896
3897             # Converted into dicts to remove duplicates
3898             captions = {
3899                 get_lang_code(sub): sub
3900                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3901             translation_languages = {
3902                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3903                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3904
3905             def process_language(container, base_url, lang_code, sub_name, query):
3906                 lang_subs = container.setdefault(lang_code, [])
3907                 for fmt in self._SUBTITLE_FORMATS:
3908                     query.update({
3909                         'fmt': fmt,
3910                     })
3911                     lang_subs.append({
3912                         'ext': fmt,
3913                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3914                         'name': sub_name,
3915                     })
3916
3917             # NB: Constructing the full subtitle dictionary is slow
3918             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
3919                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
3920             for lang_code, caption_track in captions.items():
3921                 base_url = caption_track.get('baseUrl')
3922                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3923                 if not base_url:
3924                     continue
3925                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3926                 if caption_track.get('kind') != 'asr':
3927                     if not lang_code:
3928                         continue
3929                     process_language(
3930                         subtitles, base_url, lang_code, lang_name, {})
3931                     if not caption_track.get('isTranslatable'):
3932                         continue
3933                 for trans_code, trans_name in translation_languages.items():
3934                     if not trans_code:
3935                         continue
3936                     orig_trans_code = trans_code
3937                     if caption_track.get('kind') != 'asr':
3938                         if not get_translated_subs:
3939                             continue
3940                         trans_code += f'-{lang_code}'
3941                         trans_name += format_field(lang_name, None, ' from %s')
3942                     # Add an "-orig" label to the original language so that it can be distinguished.
3943                     # The subs are returned without "-orig" as well for compatibility
3944                     if lang_code == f'a-{orig_trans_code}':
3945                         process_language(
3946                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3947                     # Setting tlang=lang returns damaged subtitles.
3948                     process_language(automatic_captions, base_url, trans_code, trans_name,
3949                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3950
3951         info['automatic_captions'] = automatic_captions
3952         info['subtitles'] = subtitles
3953
3954         parsed_url = urllib.parse.urlparse(url)
3955         for component in [parsed_url.fragment, parsed_url.query]:
3956             query = urllib.parse.parse_qs(component)
3957             for k, v in query.items():
3958                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3959                     d_k += '_time'
3960                     if d_k not in info and k in s_ks:
3961                         info[d_k] = parse_duration(query[k][0])
3962
3963         # Youtube Music Auto-generated description
3964         if video_description:
3965             mobj = re.search(
3966                 r'''(?xs)
3967                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3968                     (?P<album>[^\n]+)
3969                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3970                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3971                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3972                     .+\nAuto-generated\ by\ YouTube\.\s*$
3973                 ''', video_description)
3974             if mobj:
3975                 release_year = mobj.group('release_year')
3976                 release_date = mobj.group('release_date')
3977                 if release_date:
3978                     release_date = release_date.replace('-', '')
3979                     if not release_year:
3980                         release_year = release_date[:4]
3981                 info.update({
3982                     'album': mobj.group('album'.strip()),
3983                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3984                     'track': mobj.group('track').strip(),
3985                     'release_date': release_date,
3986                     'release_year': int_or_none(release_year),
3987                 })
3988
3989         initial_data = None
3990         if webpage:
3991             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
3992         if not initial_data:
3993             query = {'videoId': video_id}
3994             query.update(self._get_checkok_params())
3995             initial_data = self._extract_response(
3996                 item_id=video_id, ep='next', fatal=False,
3997                 ytcfg=master_ytcfg, query=query,
3998                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3999                 note='Downloading initial data API JSON')
4000
4001         info['comment_count'] = traverse_obj(initial_data, (
4002             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4003             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
4004         ), (
4005             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4006             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
4007         ), expected_type=int_or_none, get_all=False)
4008
4009         try:  # This will error if there is no livechat
4010             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4011         except (KeyError, IndexError, TypeError):
4012             pass
4013         else:
4014             info.setdefault('subtitles', {})['live_chat'] = [{
4015                 # url is needed to set cookies
4016                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4017                 'video_id': video_id,
4018                 'ext': 'json',
4019                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
4020             }]
4021
4022         if initial_data:
4023             info['chapters'] = (
4024                 self._extract_chapters_from_json(initial_data, duration)
4025                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4026                 or self._extract_chapters_from_description(video_description, duration)
4027                 or None)
4028
4029         contents = traverse_obj(
4030             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4031             expected_type=list, default=[])
4032
4033         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4034         if vpir:
4035             stl = vpir.get('superTitleLink')
4036             if stl:
4037                 stl = self._get_text(stl)
4038                 if try_get(
4039                         vpir,
4040                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4041                     info['location'] = stl
4042                 else:
4043                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4044                     if mobj:
4045                         info.update({
4046                             'series': mobj.group(1),
4047                             'season_number': int(mobj.group(2)),
4048                             'episode_number': int(mobj.group(3)),
4049                         })
4050             for tlb in (try_get(
4051                     vpir,
4052                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4053                     list) or []):
4054                 tbrs = variadic(
4055                     traverse_obj(
4056                         tlb, 'toggleButtonRenderer',
4057                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer'),
4058                         default=[]))
4059                 for tbr in tbrs:
4060                     for getter, regex in [(
4061                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4062                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4063                                 lambda x: x['accessibility'],
4064                                 lambda x: x['accessibilityData']['accessibilityData'],
4065                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4066                         label = (try_get(tbr, getter, dict) or {}).get('label')
4067                         if label:
4068                             mobj = re.match(regex, label)
4069                             if mobj:
4070                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4071                                 break
4072             sbr_tooltip = try_get(
4073                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4074             if sbr_tooltip:
4075                 like_count, dislike_count = sbr_tooltip.split(' / ')
4076                 info.update({
4077                     'like_count': str_to_int(like_count),
4078                     'dislike_count': str_to_int(dislike_count),
4079                 })
4080         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4081         if vsir:
4082             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4083             info.update({
4084                 'channel': self._get_text(vor, 'title'),
4085                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4086
4087             rows = try_get(
4088                 vsir,
4089                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4090                 list) or []
4091             multiple_songs = False
4092             for row in rows:
4093                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4094                     multiple_songs = True
4095                     break
4096             for row in rows:
4097                 mrr = row.get('metadataRowRenderer') or {}
4098                 mrr_title = mrr.get('title')
4099                 if not mrr_title:
4100                     continue
4101                 mrr_title = self._get_text(mrr, 'title')
4102                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4103                 if mrr_title == 'License':
4104                     info['license'] = mrr_contents_text
4105                 elif not multiple_songs:
4106                     if mrr_title == 'Album':
4107                         info['album'] = mrr_contents_text
4108                     elif mrr_title == 'Artist':
4109                         info['artist'] = mrr_contents_text
4110                     elif mrr_title == 'Song':
4111                         info['track'] = mrr_contents_text
4112
4113         fallbacks = {
4114             'channel': 'uploader',
4115             'channel_id': 'uploader_id',
4116             'channel_url': 'uploader_url',
4117         }
4118
4119         # The upload date for scheduled, live and past live streams / premieres in microformats
4120         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4121         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4122         upload_date = (
4123             unified_strdate(get_first(microformats, 'uploadDate'))
4124             or unified_strdate(search_meta('uploadDate')))
4125         if not upload_date or (
4126             not info.get('is_live')
4127             and not info.get('was_live')
4128             and info.get('live_status') != 'is_upcoming'
4129             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4130         ):
4131             upload_date = strftime_or_none(
4132                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4133         info['upload_date'] = upload_date
4134
4135         for to, frm in fallbacks.items():
4136             if not info.get(to):
4137                 info[to] = info.get(frm)
4138
4139         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4140             v = info.get(s_k)
4141             if v:
4142                 info[d_k] = v
4143
4144         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4145
4146         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4147                       or get_first(video_details, 'isPrivate', expected_type=bool))
4148
4149         info['availability'] = (
4150             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4151             else self._availability(
4152                 is_private=is_private,
4153                 needs_premium=(
4154                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4155                     or False if initial_data and is_private is not None else None),
4156                 needs_subscription=(
4157                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4158                     or False if initial_data and is_private is not None else None),
4159                 needs_auth=info['age_limit'] >= 18,
4160                 is_unlisted=None if is_private is None else (
4161                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4162                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4163
4164         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4165
4166         self.mark_watched(video_id, player_responses)
4167
4168         return info
4169
4170
4171 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4172
4173     @staticmethod
4174     def passthrough_smuggled_data(func):
4175         def _smuggle(entries, smuggled_data):
4176             for entry in entries:
4177                 # TODO: Convert URL to music.youtube instead.
4178                 # Do we need to passthrough any other smuggled_data?
4179                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
4180                 yield entry
4181
4182         @functools.wraps(func)
4183         def wrapper(self, url):
4184             url, smuggled_data = unsmuggle_url(url, {})
4185             if self.is_music_url(url):
4186                 smuggled_data['is_music_url'] = True
4187             info_dict = func(self, url, smuggled_data)
4188             if smuggled_data and info_dict.get('entries'):
4189                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
4190             return info_dict
4191         return wrapper
4192
4193     def _extract_channel_id(self, webpage):
4194         channel_id = self._html_search_meta(
4195             'channelId', webpage, 'channel id', default=None)
4196         if channel_id:
4197             return channel_id
4198         channel_url = self._html_search_meta(
4199             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
4200              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
4201              'twitter:app:url:googleplay'), webpage, 'channel url')
4202         return self._search_regex(
4203             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
4204             channel_url, 'channel id')
4205
4206     @staticmethod
4207     def _extract_basic_item_renderer(item):
4208         # Modified from _extract_grid_item_renderer
4209         known_basic_renderers = (
4210             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4211         )
4212         for key, renderer in item.items():
4213             if not isinstance(renderer, dict):
4214                 continue
4215             elif key in known_basic_renderers:
4216                 return renderer
4217             elif key.startswith('grid') and key.endswith('Renderer'):
4218                 return renderer
4219
4220     def _grid_entries(self, grid_renderer):
4221         for item in grid_renderer['items']:
4222             if not isinstance(item, dict):
4223                 continue
4224             renderer = self._extract_basic_item_renderer(item)
4225             if not isinstance(renderer, dict):
4226                 continue
4227             title = self._get_text(renderer, 'title')
4228
4229             # playlist
4230             playlist_id = renderer.get('playlistId')
4231             if playlist_id:
4232                 yield self.url_result(
4233                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4234                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4235                     video_title=title)
4236                 continue
4237             # video
4238             video_id = renderer.get('videoId')
4239             if video_id:
4240                 yield self._extract_video(renderer)
4241                 continue
4242             # channel
4243             channel_id = renderer.get('channelId')
4244             if channel_id:
4245                 yield self.url_result(
4246                     'https://www.youtube.com/channel/%s' % channel_id,
4247                     ie=YoutubeTabIE.ie_key(), video_title=title)
4248                 continue
4249             # generic endpoint URL support
4250             ep_url = urljoin('https://www.youtube.com/', try_get(
4251                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4252                 str))
4253             if ep_url:
4254                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4255                     if ie.suitable(ep_url):
4256                         yield self.url_result(
4257                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4258                         break
4259
4260     def _music_reponsive_list_entry(self, renderer):
4261         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4262         if video_id:
4263             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4264                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4265         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4266         if playlist_id:
4267             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4268             if video_id:
4269                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4270                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4271             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4272                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4273         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4274         if browse_id:
4275             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4276                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4277
4278     def _shelf_entries_from_content(self, shelf_renderer):
4279         content = shelf_renderer.get('content')
4280         if not isinstance(content, dict):
4281             return
4282         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4283         if renderer:
4284             # TODO: add support for nested playlists so each shelf is processed
4285             # as separate playlist
4286             # TODO: this includes only first N items
4287             yield from self._grid_entries(renderer)
4288         renderer = content.get('horizontalListRenderer')
4289         if renderer:
4290             # TODO
4291             pass
4292
4293     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4294         ep = try_get(
4295             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4296             str)
4297         shelf_url = urljoin('https://www.youtube.com', ep)
4298         if shelf_url:
4299             # Skipping links to another channels, note that checking for
4300             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4301             # will not work
4302             if skip_channels and '/channels?' in shelf_url:
4303                 return
4304             title = self._get_text(shelf_renderer, 'title')
4305             yield self.url_result(shelf_url, video_title=title)
4306         # Shelf may not contain shelf URL, fallback to extraction from content
4307         yield from self._shelf_entries_from_content(shelf_renderer)
4308
4309     def _playlist_entries(self, video_list_renderer):
4310         for content in video_list_renderer['contents']:
4311             if not isinstance(content, dict):
4312                 continue
4313             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4314             if not isinstance(renderer, dict):
4315                 continue
4316             video_id = renderer.get('videoId')
4317             if not video_id:
4318                 continue
4319             yield self._extract_video(renderer)
4320
4321     def _rich_entries(self, rich_grid_renderer):
4322         renderer = traverse_obj(
4323             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4324         video_id = renderer.get('videoId')
4325         if not video_id:
4326             return
4327         yield self._extract_video(renderer)
4328
4329     def _video_entry(self, video_renderer):
4330         video_id = video_renderer.get('videoId')
4331         if video_id:
4332             return self._extract_video(video_renderer)
4333
4334     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4335         url = urljoin('https://youtube.com', traverse_obj(
4336             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4337         if url:
4338             return self.url_result(
4339                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4340
4341     def _post_thread_entries(self, post_thread_renderer):
4342         post_renderer = try_get(
4343             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4344         if not post_renderer:
4345             return
4346         # video attachment
4347         video_renderer = try_get(
4348             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4349         video_id = video_renderer.get('videoId')
4350         if video_id:
4351             entry = self._extract_video(video_renderer)
4352             if entry:
4353                 yield entry
4354         # playlist attachment
4355         playlist_id = try_get(
4356             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4357         if playlist_id:
4358             yield self.url_result(
4359                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4360                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4361         # inline video links
4362         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4363         for run in runs:
4364             if not isinstance(run, dict):
4365                 continue
4366             ep_url = try_get(
4367                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4368             if not ep_url:
4369                 continue
4370             if not YoutubeIE.suitable(ep_url):
4371                 continue
4372             ep_video_id = YoutubeIE._match_id(ep_url)
4373             if video_id == ep_video_id:
4374                 continue
4375             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4376
4377     def _post_thread_continuation_entries(self, post_thread_continuation):
4378         contents = post_thread_continuation.get('contents')
4379         if not isinstance(contents, list):
4380             return
4381         for content in contents:
4382             renderer = content.get('backstagePostThreadRenderer')
4383             if isinstance(renderer, dict):
4384                 yield from self._post_thread_entries(renderer)
4385                 continue
4386             renderer = content.get('videoRenderer')
4387             if isinstance(renderer, dict):
4388                 yield self._video_entry(renderer)
4389
4390     r''' # unused
4391     def _rich_grid_entries(self, contents):
4392         for content in contents:
4393             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4394             if video_renderer:
4395                 entry = self._video_entry(video_renderer)
4396                 if entry:
4397                     yield entry
4398     '''
4399
4400     def _report_history_entries(self, renderer):
4401         for url in traverse_obj(renderer, (
4402                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4403                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4404                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4405             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4406
4407     def _extract_entries(self, parent_renderer, continuation_list):
4408         # continuation_list is modified in-place with continuation_list = [continuation_token]
4409         continuation_list[:] = [None]
4410         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4411         for content in contents:
4412             if not isinstance(content, dict):
4413                 continue
4414             is_renderer = traverse_obj(
4415                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4416                 expected_type=dict)
4417             if not is_renderer:
4418                 if content.get('richItemRenderer'):
4419                     for entry in self._rich_entries(content['richItemRenderer']):
4420                         yield entry
4421                     continuation_list[0] = self._extract_continuation(parent_renderer)
4422                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4423                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4424                     yield from self._report_history_entries(table)
4425                     continuation_list[0] = self._extract_continuation(table)
4426                 continue
4427
4428             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4429             for isr_content in isr_contents:
4430                 if not isinstance(isr_content, dict):
4431                     continue
4432
4433                 known_renderers = {
4434                     'playlistVideoListRenderer': self._playlist_entries,
4435                     'gridRenderer': self._grid_entries,
4436                     'reelShelfRenderer': self._grid_entries,
4437                     'shelfRenderer': self._shelf_entries,
4438                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4439                     'backstagePostThreadRenderer': self._post_thread_entries,
4440                     'videoRenderer': lambda x: [self._video_entry(x)],
4441                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4442                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4443                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4444                 }
4445                 for key, renderer in isr_content.items():
4446                     if key not in known_renderers:
4447                         continue
4448                     for entry in known_renderers[key](renderer):
4449                         if entry:
4450                             yield entry
4451                     continuation_list[0] = self._extract_continuation(renderer)
4452                     break
4453
4454             if not continuation_list[0]:
4455                 continuation_list[0] = self._extract_continuation(is_renderer)
4456
4457         if not continuation_list[0]:
4458             continuation_list[0] = self._extract_continuation(parent_renderer)
4459
4460     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4461         continuation_list = [None]
4462         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4463         tab_content = try_get(tab, lambda x: x['content'], dict)
4464         if not tab_content:
4465             return
4466         parent_renderer = (
4467             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4468             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4469         yield from extract_entries(parent_renderer)
4470         continuation = continuation_list[0]
4471
4472         for page_num in itertools.count(1):
4473             if not continuation:
4474                 break
4475             headers = self.generate_api_headers(
4476                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4477             response = self._extract_response(
4478                 item_id=f'{item_id} page {page_num}',
4479                 query=continuation, headers=headers, ytcfg=ytcfg,
4480                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4481
4482             if not response:
4483                 break
4484             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4485             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4486             visitor_data = self._extract_visitor_data(response) or visitor_data
4487
4488             known_renderers = {
4489                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4490                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4491                 'gridVideoRenderer': (self._grid_entries, 'items'),
4492                 'gridChannelRenderer': (self._grid_entries, 'items'),
4493                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4494                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4495                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4496                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4497                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4498                 'playlistVideoListContinuation': (self._playlist_entries, None),
4499                 'gridContinuation': (self._grid_entries, None),
4500                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4501                 'sectionListContinuation': (extract_entries, None),  # for feeds
4502             }
4503
4504             continuation_items = traverse_obj(response, (
4505                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4506                 'appendContinuationItemsAction', 'continuationItems'
4507             ), 'continuationContents', get_all=False)
4508             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4509
4510             video_items_renderer = None
4511             for key in continuation_item.keys():
4512                 if key not in known_renderers:
4513                     continue
4514                 func, parent_key = known_renderers[key]
4515                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4516                 continuation_list = [None]
4517                 yield from func(video_items_renderer)
4518                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4519
4520             if not video_items_renderer:
4521                 break
4522
4523     @staticmethod
4524     def _extract_selected_tab(tabs, fatal=True):
4525         for tab in tabs:
4526             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4527             if renderer.get('selected') is True:
4528                 return renderer
4529         else:
4530             if fatal:
4531                 raise ExtractorError('Unable to find selected tab')
4532
4533     def _extract_uploader(self, data):
4534         uploader = {}
4535         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4536         owner = try_get(
4537             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4538         if owner:
4539             owner_text = owner.get('text')
4540             uploader['uploader'] = self._search_regex(
4541                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4542             uploader['uploader_id'] = try_get(
4543                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], str)
4544             uploader['uploader_url'] = urljoin(
4545                 'https://www.youtube.com/',
4546                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], str))
4547         return filter_dict(uploader)
4548
4549     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4550         playlist_id = title = description = channel_url = channel_name = channel_id = None
4551         tags = []
4552
4553         selected_tab = self._extract_selected_tab(tabs)
4554         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4555         renderer = try_get(
4556             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4557         if renderer:
4558             channel_name = renderer.get('title')
4559             channel_url = renderer.get('channelUrl')
4560             channel_id = renderer.get('externalId')
4561         else:
4562             renderer = try_get(
4563                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4564
4565         if renderer:
4566             title = renderer.get('title')
4567             description = renderer.get('description', '')
4568             playlist_id = channel_id
4569             tags = renderer.get('keywords', '').split()
4570
4571         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4572         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4573         def _get_uncropped(url):
4574             return url_or_none((url or '').split('=')[0] + '=s0')
4575
4576         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4577         if avatar_thumbnails:
4578             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4579             if uncropped_avatar:
4580                 avatar_thumbnails.append({
4581                     'url': uncropped_avatar,
4582                     'id': 'avatar_uncropped',
4583                     'preference': 1
4584                 })
4585
4586         channel_banners = self._extract_thumbnails(
4587             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4588         for banner in channel_banners:
4589             banner['preference'] = -10
4590
4591         if channel_banners:
4592             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4593             if uncropped_banner:
4594                 channel_banners.append({
4595                     'url': uncropped_banner,
4596                     'id': 'banner_uncropped',
4597                     'preference': -5
4598                 })
4599
4600         primary_thumbnails = self._extract_thumbnails(
4601             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4602
4603         if playlist_id is None:
4604             playlist_id = item_id
4605
4606         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4607         last_updated_unix = self._parse_time_text(self._get_text(playlist_stats, 2))
4608         if title is None:
4609             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4610         title += format_field(selected_tab, 'title', ' - %s')
4611         title += format_field(selected_tab, 'expandedText', ' - %s')
4612
4613         metadata = {
4614             'playlist_id': playlist_id,
4615             'playlist_title': title,
4616             'playlist_description': description,
4617             'uploader': channel_name,
4618             'uploader_id': channel_id,
4619             'uploader_url': channel_url,
4620             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4621             'tags': tags,
4622             'view_count': self._get_count(playlist_stats, 1),
4623             'availability': self._extract_availability(data),
4624             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4625             'playlist_count': self._get_count(playlist_stats, 0),
4626             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4627         }
4628         if not channel_id:
4629             metadata.update(self._extract_uploader(data))
4630         metadata.update({
4631             'channel': metadata['uploader'],
4632             'channel_id': metadata['uploader_id'],
4633             'channel_url': metadata['uploader_url']})
4634         return self.playlist_result(
4635             self._entries(
4636                 selected_tab, playlist_id, ytcfg,
4637                 self._extract_account_syncid(ytcfg, data),
4638                 self._extract_visitor_data(data, ytcfg)),
4639             **metadata)
4640
4641     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4642         first_id = last_id = response = None
4643         for page_num in itertools.count(1):
4644             videos = list(self._playlist_entries(playlist))
4645             if not videos:
4646                 return
4647             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4648             if start >= len(videos):
4649                 return
4650             yield from videos[start:]
4651             first_id = first_id or videos[0]['id']
4652             last_id = videos[-1]['id']
4653             watch_endpoint = try_get(
4654                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4655             headers = self.generate_api_headers(
4656                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4657                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4658             query = {
4659                 'playlistId': playlist_id,
4660                 'videoId': watch_endpoint.get('videoId') or last_id,
4661                 'index': watch_endpoint.get('index') or len(videos),
4662                 'params': watch_endpoint.get('params') or 'OAE%3D'
4663             }
4664             response = self._extract_response(
4665                 item_id='%s page %d' % (playlist_id, page_num),
4666                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4667                 check_get_keys='contents'
4668             )
4669             playlist = try_get(
4670                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4671
4672     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4673         title = playlist.get('title') or try_get(
4674             data, lambda x: x['titleText']['simpleText'], str)
4675         playlist_id = playlist.get('playlistId') or item_id
4676
4677         # Delegating everything except mix playlists to regular tab-based playlist URL
4678         playlist_url = urljoin(url, try_get(
4679             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4680             str))
4681
4682         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4683         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4684         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4685
4686         if playlist_url and playlist_url != url and not is_known_unviewable:
4687             return self.url_result(
4688                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4689                 video_title=title)
4690
4691         return self.playlist_result(
4692             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4693             playlist_id=playlist_id, playlist_title=title)
4694
4695     def _extract_availability(self, data):
4696         """
4697         Gets the availability of a given playlist/tab.
4698         Note: Unless YouTube tells us explicitly, we do not assume it is public
4699         @param data: response
4700         """
4701         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4702
4703         player_header_privacy = traverse_obj(
4704             data, ('header', 'playlistHeaderRenderer', 'privacy'), expected_type=str)
4705
4706         badges = self._extract_badges(renderer)
4707
4708         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4709         privacy_setting_icon = traverse_obj(
4710             renderer, (
4711                 'privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
4712                 lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
4713             get_all=False, expected_type=str)
4714
4715         return (
4716             'public' if (
4717                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4718                 or player_header_privacy == 'PUBLIC'
4719                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
4720             else self._availability(
4721                 is_private=(
4722                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4723                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
4724                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
4725                 is_unlisted=(
4726                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4727                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
4728                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None else None),
4729                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
4730                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
4731                 needs_auth=False))
4732
4733     @staticmethod
4734     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4735         sidebar_renderer = try_get(
4736             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4737         for item in sidebar_renderer:
4738             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4739             if renderer:
4740                 return renderer
4741
4742     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4743         """
4744         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4745         """
4746         browse_id = params = None
4747         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4748         if not renderer:
4749             return
4750         menu_renderer = try_get(
4751             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4752         for menu_item in menu_renderer:
4753             if not isinstance(menu_item, dict):
4754                 continue
4755             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4756             text = try_get(
4757                 nav_item_renderer, lambda x: x['text']['simpleText'], str)
4758             if not text or text.lower() != 'show unavailable videos':
4759                 continue
4760             browse_endpoint = try_get(
4761                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4762             browse_id = browse_endpoint.get('browseId')
4763             params = browse_endpoint.get('params')
4764             break
4765
4766         headers = self.generate_api_headers(
4767             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4768             visitor_data=self._extract_visitor_data(data, ytcfg))
4769         query = {
4770             'params': params or 'wgYCCAA=',
4771             'browseId': browse_id or 'VL%s' % item_id
4772         }
4773         return self._extract_response(
4774             item_id=item_id, headers=headers, query=query,
4775             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4776             note='Downloading API JSON with unavailable videos')
4777
4778     @functools.cached_property
4779     def skip_webpage(self):
4780         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4781
4782     def _extract_webpage(self, url, item_id, fatal=True):
4783         webpage, data = None, None
4784         for retry in self.RetryManager(fatal=fatal):
4785             try:
4786                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
4787                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4788             except ExtractorError as e:
4789                 if isinstance(e.cause, network_exceptions):
4790                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4791                         retry.error = e
4792                         continue
4793                 self._error_or_warning(e, fatal=fatal)
4794                 break
4795
4796             try:
4797                 self._extract_and_report_alerts(data)
4798             except ExtractorError as e:
4799                 self._error_or_warning(e, fatal=fatal)
4800                 break
4801
4802             # Sometimes youtube returns a webpage with incomplete ytInitialData
4803             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4804             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
4805                 retry.error = ExtractorError('Incomplete yt initial data received')
4806                 continue
4807
4808         return webpage, data
4809
4810     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4811         """Use if failed to extract ytcfg (and data) from initial webpage"""
4812         if not ytcfg and self.is_authenticated:
4813             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4814             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4815                 raise ExtractorError(
4816                     f'{msg}. If you are not downloading private content, or '
4817                     'your cookies are only for the first account and channel,'
4818                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4819                     expected=True)
4820             self.report_warning(msg, only_once=True)
4821
4822     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4823         data = None
4824         if not self.skip_webpage:
4825             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4826             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4827             # Reject webpage data if redirected to home page without explicitly requesting
4828             selected_tab = self._extract_selected_tab(traverse_obj(
4829                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4830             if (url != 'https://www.youtube.com/feed/recommended'
4831                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4832                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4833                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4834                 if fatal:
4835                     raise ExtractorError(msg, expected=True)
4836                 self.report_warning(msg, only_once=True)
4837         if not data:
4838             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4839             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4840         return data, ytcfg
4841
4842     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4843         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4844         resolve_response = self._extract_response(
4845             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4846             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4847         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4848         for ep_key, ep in endpoints.items():
4849             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4850             if params:
4851                 return self._extract_response(
4852                     item_id=item_id, query=params, ep=ep, headers=headers,
4853                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4854                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4855         err_note = 'Failed to resolve url (does the playlist exist?)'
4856         if fatal:
4857             raise ExtractorError(err_note, expected=True)
4858         self.report_warning(err_note, item_id)
4859
4860     _SEARCH_PARAMS = None
4861
4862     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4863         data = {'query': query}
4864         if params is NO_DEFAULT:
4865             params = self._SEARCH_PARAMS
4866         if params:
4867             data['params'] = params
4868
4869         content_keys = (
4870             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4871             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4872             # ytmusic search
4873             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4874             ('continuationContents', ),
4875         )
4876         display_id = f'query "{query}"'
4877         check_get_keys = tuple({keys[0] for keys in content_keys})
4878         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4879         self._report_playlist_authcheck(ytcfg, fatal=False)
4880
4881         continuation_list = [None]
4882         search = None
4883         for page_num in itertools.count(1):
4884             data.update(continuation_list[0] or {})
4885             headers = self.generate_api_headers(
4886                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4887             search = self._extract_response(
4888                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4889                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4890             slr_contents = traverse_obj(search, *content_keys)
4891             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4892             if not continuation_list[0]:
4893                 break
4894
4895
4896 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4897     IE_DESC = 'YouTube Tabs'
4898     _VALID_URL = r'''(?x:
4899         https?://
4900             (?:\w+\.)?
4901             (?:
4902                 youtube(?:kids)?\.com|
4903                 %(invidious)s
4904             )/
4905             (?:
4906                 (?P<channel_type>channel|c|user|browse)/|
4907                 (?P<not_channel>
4908                     feed/|hashtag/|
4909                     (?:playlist|watch)\?.*?\blist=
4910                 )|
4911                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4912             )
4913             (?P<id>[^/?\#&]+)
4914     )''' % {
4915         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4916         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4917     }
4918     IE_NAME = 'youtube:tab'
4919
4920     _TESTS = [{
4921         'note': 'playlists, multipage',
4922         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4923         'playlist_mincount': 94,
4924         'info_dict': {
4925             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4926             'title': 'Igor Kleiner - Playlists',
4927             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4928             'uploader': 'Igor Kleiner',
4929             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4930             'channel': 'Igor Kleiner',
4931             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4932             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4933             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4934             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4935             'channel_follower_count': int
4936         },
4937     }, {
4938         'note': 'playlists, multipage, different order',
4939         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4940         'playlist_mincount': 94,
4941         'info_dict': {
4942             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4943             'title': 'Igor Kleiner - Playlists',
4944             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4945             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4946             'uploader': 'Igor Kleiner',
4947             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4948             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4949             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4950             'channel': 'Igor Kleiner',
4951             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4952             'channel_follower_count': int
4953         },
4954     }, {
4955         'note': 'playlists, series',
4956         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4957         'playlist_mincount': 5,
4958         'info_dict': {
4959             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4960             'title': '3Blue1Brown - Playlists',
4961             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4962             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4963             'uploader': '3Blue1Brown',
4964             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4965             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4966             'channel': '3Blue1Brown',
4967             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4968             'tags': ['Mathematics'],
4969             'channel_follower_count': int
4970         },
4971     }, {
4972         'note': 'playlists, singlepage',
4973         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4974         'playlist_mincount': 4,
4975         'info_dict': {
4976             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4977             'title': 'ThirstForScience - Playlists',
4978             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4979             'uploader': 'ThirstForScience',
4980             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4981             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4982             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4983             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4984             'tags': 'count:13',
4985             'channel': 'ThirstForScience',
4986             'channel_follower_count': int
4987         }
4988     }, {
4989         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4990         'only_matching': True,
4991     }, {
4992         'note': 'basic, single video playlist',
4993         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4994         'info_dict': {
4995             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4996             'uploader': 'Sergey M.',
4997             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4998             'title': 'youtube-dl public playlist',
4999             'description': '',
5000             'tags': [],
5001             'view_count': int,
5002             'modified_date': '20201130',
5003             'channel': 'Sergey M.',
5004             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5005             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5006             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5007             'availability': 'public',
5008         },
5009         'playlist_count': 1,
5010     }, {
5011         'note': 'empty playlist',
5012         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5013         'info_dict': {
5014             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5015             'uploader': 'Sergey M.',
5016             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5017             'title': 'youtube-dl empty playlist',
5018             'tags': [],
5019             'channel': 'Sergey M.',
5020             'description': '',
5021             'modified_date': '20160902',
5022             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5023             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5024             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5025             'availability': 'public',
5026         },
5027         'playlist_count': 0,
5028     }, {
5029         'note': 'Home tab',
5030         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5031         'info_dict': {
5032             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5033             'title': 'lex will - Home',
5034             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5035             'uploader': 'lex will',
5036             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5037             'channel': 'lex will',
5038             'tags': ['bible', 'history', 'prophesy'],
5039             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5040             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5041             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5042             'channel_follower_count': int
5043         },
5044         'playlist_mincount': 2,
5045     }, {
5046         'note': 'Videos tab',
5047         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5048         'info_dict': {
5049             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5050             'title': 'lex will - Videos',
5051             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5052             'uploader': 'lex will',
5053             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5054             'tags': ['bible', 'history', 'prophesy'],
5055             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5056             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5057             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5058             'channel': 'lex will',
5059             'channel_follower_count': int
5060         },
5061         'playlist_mincount': 975,
5062     }, {
5063         'note': 'Videos tab, sorted by popular',
5064         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5065         'info_dict': {
5066             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5067             'title': 'lex will - Videos',
5068             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5069             'uploader': 'lex will',
5070             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5071             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5072             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5073             'channel': 'lex will',
5074             'tags': ['bible', 'history', 'prophesy'],
5075             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5076             'channel_follower_count': int
5077         },
5078         'playlist_mincount': 199,
5079     }, {
5080         'note': 'Playlists tab',
5081         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5082         'info_dict': {
5083             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5084             'title': 'lex will - Playlists',
5085             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5086             'uploader': 'lex will',
5087             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5088             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5089             'channel': 'lex will',
5090             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5091             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5092             'tags': ['bible', 'history', 'prophesy'],
5093             'channel_follower_count': int
5094         },
5095         'playlist_mincount': 17,
5096     }, {
5097         'note': 'Community tab',
5098         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5099         'info_dict': {
5100             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5101             'title': 'lex will - Community',
5102             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5103             'uploader': 'lex will',
5104             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5105             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5106             'channel': 'lex will',
5107             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5108             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5109             'tags': ['bible', 'history', 'prophesy'],
5110             'channel_follower_count': int
5111         },
5112         'playlist_mincount': 18,
5113     }, {
5114         'note': 'Channels tab',
5115         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5116         'info_dict': {
5117             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5118             'title': 'lex will - Channels',
5119             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5120             'uploader': 'lex will',
5121             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5122             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5123             'channel': 'lex will',
5124             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5125             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5126             'tags': ['bible', 'history', 'prophesy'],
5127             'channel_follower_count': int
5128         },
5129         'playlist_mincount': 12,
5130     }, {
5131         'note': 'Search tab',
5132         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5133         'playlist_mincount': 40,
5134         'info_dict': {
5135             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5136             'title': '3Blue1Brown - Search - linear algebra',
5137             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5138             'uploader': '3Blue1Brown',
5139             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5140             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5141             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5142             'tags': ['Mathematics'],
5143             'channel': '3Blue1Brown',
5144             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5145             'channel_follower_count': int
5146         },
5147     }, {
5148         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5149         'only_matching': True,
5150     }, {
5151         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5152         'only_matching': True,
5153     }, {
5154         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5155         'only_matching': True,
5156     }, {
5157         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5158         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5159         'info_dict': {
5160             'title': '29C3: Not my department',
5161             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5162             'uploader': 'Christiaan008',
5163             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5164             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5165             'tags': [],
5166             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5167             'view_count': int,
5168             'modified_date': '20150605',
5169             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5170             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5171             'channel': 'Christiaan008',
5172             'availability': 'public',
5173         },
5174         'playlist_count': 96,
5175     }, {
5176         'note': 'Large playlist',
5177         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5178         'info_dict': {
5179             'title': 'Uploads from Cauchemar',
5180             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5181             'uploader': 'Cauchemar',
5182             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5183             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5184             'tags': [],
5185             'modified_date': r're:\d{8}',
5186             'channel': 'Cauchemar',
5187             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5188             'view_count': int,
5189             'description': '',
5190             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5191             'availability': 'public',
5192         },
5193         'playlist_mincount': 1123,
5194         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5195     }, {
5196         'note': 'even larger playlist, 8832 videos',
5197         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5198         'only_matching': True,
5199     }, {
5200         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5201         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5202         'info_dict': {
5203             'title': 'Uploads from Interstellar Movie',
5204             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5205             'uploader': 'Interstellar Movie',
5206             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5207             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5208             'tags': [],
5209             'view_count': int,
5210             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5211             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5212             'channel': 'Interstellar Movie',
5213             'description': '',
5214             'modified_date': r're:\d{8}',
5215             'availability': 'public',
5216         },
5217         'playlist_mincount': 21,
5218     }, {
5219         'note': 'Playlist with "show unavailable videos" button',
5220         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5221         'info_dict': {
5222             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5223             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5224             'uploader': 'Phim Siêu Nhân Nhật Bản',
5225             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5226             'view_count': int,
5227             'channel': 'Phim Siêu Nhân Nhật Bản',
5228             'tags': [],
5229             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5230             'description': '',
5231             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5232             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5233             'modified_date': r're:\d{8}',
5234             'availability': 'public',
5235         },
5236         'playlist_mincount': 200,
5237         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5238     }, {
5239         'note': 'Playlist with unavailable videos in page 7',
5240         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5241         'info_dict': {
5242             'title': 'Uploads from BlankTV',
5243             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5244             'uploader': 'BlankTV',
5245             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5246             'channel': 'BlankTV',
5247             'channel_url': 'https://www.youtube.com/c/blanktv',
5248             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5249             'view_count': int,
5250             'tags': [],
5251             'uploader_url': 'https://www.youtube.com/c/blanktv',
5252             'modified_date': r're:\d{8}',
5253             'description': '',
5254             'availability': 'public',
5255         },
5256         'playlist_mincount': 1000,
5257         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5258     }, {
5259         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5260         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5261         'info_dict': {
5262             'title': 'Data Analysis with Dr Mike Pound',
5263             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5264             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5265             'uploader': 'Computerphile',
5266             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5267             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5268             'tags': [],
5269             'view_count': int,
5270             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5271             'channel_url': 'https://www.youtube.com/user/Computerphile',
5272             'channel': 'Computerphile',
5273             'availability': 'public',
5274         },
5275         'playlist_mincount': 11,
5276     }, {
5277         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5278         'only_matching': True,
5279     }, {
5280         'note': 'Playlist URL that does not actually serve a playlist',
5281         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5282         'info_dict': {
5283             'id': 'FqZTN594JQw',
5284             'ext': 'webm',
5285             'title': "Smiley's People 01 detective, Adventure Series, Action",
5286             'uploader': 'STREEM',
5287             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5288             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5289             'upload_date': '20150526',
5290             'license': 'Standard YouTube License',
5291             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5292             'categories': ['People & Blogs'],
5293             'tags': list,
5294             'view_count': int,
5295             'like_count': int,
5296         },
5297         'params': {
5298             'skip_download': True,
5299         },
5300         'skip': 'This video is not available.',
5301         'add_ie': [YoutubeIE.ie_key()],
5302     }, {
5303         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5304         'only_matching': True,
5305     }, {
5306         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5307         'only_matching': True,
5308     }, {
5309         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5310         'info_dict': {
5311             'id': 'Wq15eF5vCbI',  # This will keep changing
5312             'ext': 'mp4',
5313             'title': str,
5314             'uploader': 'Sky News',
5315             'uploader_id': 'skynews',
5316             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5317             'upload_date': r're:\d{8}',
5318             'description': str,
5319             'categories': ['News & Politics'],
5320             'tags': list,
5321             'like_count': int,
5322             'release_timestamp': 1642502819,
5323             'channel': 'Sky News',
5324             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5325             'age_limit': 0,
5326             'view_count': int,
5327             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5328             'playable_in_embed': True,
5329             'release_date': '20220118',
5330             'availability': 'public',
5331             'live_status': 'is_live',
5332             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5333             'channel_follower_count': int
5334         },
5335         'params': {
5336             'skip_download': True,
5337         },
5338         'expected_warnings': ['Ignoring subtitle tracks found in '],
5339     }, {
5340         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5341         'info_dict': {
5342             'id': 'a48o2S1cPoo',
5343             'ext': 'mp4',
5344             'title': 'The Young Turks - Live Main Show',
5345             'uploader': 'The Young Turks',
5346             'uploader_id': 'TheYoungTurks',
5347             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5348             'upload_date': '20150715',
5349             'license': 'Standard YouTube License',
5350             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5351             'categories': ['News & Politics'],
5352             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5353             'like_count': int,
5354         },
5355         'params': {
5356             'skip_download': True,
5357         },
5358         'only_matching': True,
5359     }, {
5360         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5361         'only_matching': True,
5362     }, {
5363         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5364         'only_matching': True,
5365     }, {
5366         'note': 'A channel that is not live. Should raise error',
5367         'url': 'https://www.youtube.com/user/numberphile/live',
5368         'only_matching': True,
5369     }, {
5370         'url': 'https://www.youtube.com/feed/trending',
5371         'only_matching': True,
5372     }, {
5373         'url': 'https://www.youtube.com/feed/library',
5374         'only_matching': True,
5375     }, {
5376         'url': 'https://www.youtube.com/feed/history',
5377         'only_matching': True,
5378     }, {
5379         'url': 'https://www.youtube.com/feed/subscriptions',
5380         'only_matching': True,
5381     }, {
5382         'url': 'https://www.youtube.com/feed/watch_later',
5383         'only_matching': True,
5384     }, {
5385         'note': 'Recommended - redirects to home page.',
5386         'url': 'https://www.youtube.com/feed/recommended',
5387         'only_matching': True,
5388     }, {
5389         'note': 'inline playlist with not always working continuations',
5390         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5391         'only_matching': True,
5392     }, {
5393         'url': 'https://www.youtube.com/course',
5394         'only_matching': True,
5395     }, {
5396         'url': 'https://www.youtube.com/zsecurity',
5397         'only_matching': True,
5398     }, {
5399         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5400         'only_matching': True,
5401     }, {
5402         'url': 'https://www.youtube.com/TheYoungTurks/live',
5403         'only_matching': True,
5404     }, {
5405         'url': 'https://www.youtube.com/hashtag/cctv9',
5406         'info_dict': {
5407             'id': 'cctv9',
5408             'title': '#cctv9',
5409             'tags': [],
5410         },
5411         'playlist_mincount': 350,
5412     }, {
5413         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5414         'only_matching': True,
5415     }, {
5416         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5417         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5418         'only_matching': True
5419     }, {
5420         'note': '/browse/ should redirect to /channel/',
5421         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5422         'only_matching': True
5423     }, {
5424         'note': 'VLPL, should redirect to playlist?list=PL...',
5425         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5426         'info_dict': {
5427             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5428             'uploader': 'NoCopyrightSounds',
5429             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5430             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5431             'title': 'NCS : All Releases 💿',
5432             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5433             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5434             'modified_date': r're:\d{8}',
5435             'view_count': int,
5436             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5437             'tags': [],
5438             'channel': 'NoCopyrightSounds',
5439             'availability': 'public',
5440         },
5441         'playlist_mincount': 166,
5442         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5443     }, {
5444         'note': 'Topic, should redirect to playlist?list=UU...',
5445         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5446         'info_dict': {
5447             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5448             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5449             'title': 'Uploads from Royalty Free Music - Topic',
5450             'uploader': 'Royalty Free Music - Topic',
5451             'tags': [],
5452             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5453             'channel': 'Royalty Free Music - Topic',
5454             'view_count': int,
5455             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5456             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5457             'modified_date': r're:\d{8}',
5458             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5459             'description': '',
5460             'availability': 'public',
5461         },
5462         'expected_warnings': [
5463             'The URL does not have a videos tab',
5464             r'[Uu]navailable videos (are|will be) hidden',
5465         ],
5466         'playlist_mincount': 101,
5467     }, {
5468         'note': 'Topic without a UU playlist',
5469         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5470         'info_dict': {
5471             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5472             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5473             'tags': [],
5474         },
5475         'expected_warnings': [
5476             'the playlist redirect gave error',
5477         ],
5478         'playlist_mincount': 9,
5479     }, {
5480         'note': 'Youtube music Album',
5481         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5482         'info_dict': {
5483             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5484             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5485             'tags': [],
5486             'view_count': int,
5487             'description': '',
5488             'availability': 'unlisted',
5489             'modified_date': r're:\d{8}',
5490         },
5491         'playlist_count': 50,
5492     }, {
5493         'note': 'unlisted single video playlist',
5494         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5495         'info_dict': {
5496             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5497             'uploader': 'colethedj',
5498             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5499             'title': 'yt-dlp unlisted playlist test',
5500             'availability': 'unlisted',
5501             'tags': [],
5502             'modified_date': '20220418',
5503             'channel': 'colethedj',
5504             'view_count': int,
5505             'description': '',
5506             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5507             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5508             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5509         },
5510         'playlist_count': 1,
5511     }, {
5512         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5513         'url': 'https://www.youtube.com/feed/recommended',
5514         'info_dict': {
5515             'id': 'recommended',
5516             'title': 'recommended',
5517             'tags': [],
5518         },
5519         'playlist_mincount': 50,
5520         'params': {
5521             'skip_download': True,
5522             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5523         },
5524     }, {
5525         'note': 'API Fallback: /videos tab, sorted by oldest first',
5526         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5527         'info_dict': {
5528             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5529             'title': 'Cody\'sLab - Videos',
5530             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5531             'uploader': 'Cody\'sLab',
5532             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5533             'channel': 'Cody\'sLab',
5534             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5535             'tags': [],
5536             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5537             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5538             'channel_follower_count': int
5539         },
5540         'playlist_mincount': 650,
5541         'params': {
5542             'skip_download': True,
5543             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5544         },
5545     }, {
5546         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5547         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5548         'info_dict': {
5549             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5550             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5551             'title': 'Uploads from Royalty Free Music - Topic',
5552             'uploader': 'Royalty Free Music - Topic',
5553             'modified_date': r're:\d{8}',
5554             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5555             'description': '',
5556             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5557             'tags': [],
5558             'channel': 'Royalty Free Music - Topic',
5559             'view_count': int,
5560             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5561             'availability': 'public',
5562         },
5563         'expected_warnings': [
5564             'does not have a videos tab',
5565             r'[Uu]navailable videos (are|will be) hidden',
5566         ],
5567         'playlist_mincount': 101,
5568         'params': {
5569             'skip_download': True,
5570             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5571         },
5572     }, {
5573         'note': 'non-standard redirect to regional channel',
5574         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5575         'only_matching': True
5576     }, {
5577         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5578         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5579         'info_dict': {
5580             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5581             'modified_date': '20220407',
5582             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5583             'tags': [],
5584             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5585             'uploader': 'pukkandan',
5586             'availability': 'unlisted',
5587             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5588             'channel': 'pukkandan',
5589             'description': 'Test for collaborative playlist',
5590             'title': 'yt-dlp test - collaborative playlist',
5591             'view_count': int,
5592             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5593         },
5594         'playlist_mincount': 2
5595     }, {
5596         'note': 'translated tab name',
5597         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5598         'info_dict': {
5599             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5600             'tags': [],
5601             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5602             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5603             'description': '',
5604             'title': 'cole-dlp-test-acc - 再生リスト',
5605             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5606             'uploader': 'cole-dlp-test-acc',
5607             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5608             'channel': 'cole-dlp-test-acc',
5609         },
5610         'playlist_mincount': 1,
5611         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5612         'expected_warnings': ['Preferring "ja"'],
5613     }, {
5614         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5615         'note': 'preferred lang set with playlist with translated video titles',
5616         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5617         'info_dict': {
5618             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5619             'tags': [],
5620             'view_count': int,
5621             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5622             'uploader': 'cole-dlp-test-acc',
5623             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5624             'channel': 'cole-dlp-test-acc',
5625             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5626             'description': 'test',
5627             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5628             'title': 'dlp test playlist',
5629             'availability': 'public',
5630         },
5631         'playlist_mincount': 1,
5632         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5633         'expected_warnings': ['Preferring "ja"'],
5634     }, {
5635         # shorts audio pivot for 2GtVksBMYFM.
5636         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5637         'info_dict': {
5638             'id': 'sfv_audio_pivot',
5639             'title': 'sfv_audio_pivot',
5640             'tags': [],
5641         },
5642         'playlist_mincount': 50,
5643
5644     }]
5645
5646     @classmethod
5647     def suitable(cls, url):
5648         return False if YoutubeIE.suitable(url) else super().suitable(url)
5649
5650     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5651
5652     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5653     def _real_extract(self, url, smuggled_data):
5654         item_id = self._match_id(url)
5655         url = urllib.parse.urlunparse(
5656             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5657         compat_opts = self.get_param('compat_opts', [])
5658
5659         def get_mobj(url):
5660             mobj = self._URL_RE.match(url).groupdict()
5661             mobj.update((k, '') for k, v in mobj.items() if v is None)
5662             return mobj
5663
5664         mobj, redirect_warning = get_mobj(url), None
5665         # Youtube returns incomplete data if tabname is not lower case
5666         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5667         if is_channel:
5668             if smuggled_data.get('is_music_url'):
5669                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5670                     item_id = item_id[2:]
5671                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5672                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5673                     mdata = self._extract_tab_endpoint(
5674                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5675                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5676                                         get_all=False, expected_type=str)
5677                     if not murl:
5678                         raise ExtractorError('Failed to resolve album to playlist')
5679                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5680                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5681                     pre = f'https://www.youtube.com/channel/{item_id}'
5682
5683         original_tab_name = tab
5684         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5685             # Home URLs should redirect to /videos/
5686             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5687                                 'To download only the videos in the home page, add a "/featured" to the URL')
5688             tab = '/videos'
5689
5690         url = ''.join((pre, tab, post))
5691         mobj = get_mobj(url)
5692
5693         # Handle both video/playlist URLs
5694         qs = parse_qs(url)
5695         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5696
5697         if not video_id and mobj['not_channel'].startswith('watch'):
5698             if not playlist_id:
5699                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5700                 raise ExtractorError('Unable to recognize tab page')
5701             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5702             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5703             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5704             mobj = get_mobj(url)
5705
5706         if video_id and playlist_id:
5707             if self.get_param('noplaylist'):
5708                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5709                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5710                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5711             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5712
5713         data, ytcfg = self._extract_data(url, item_id)
5714
5715         # YouTube may provide a non-standard redirect to the regional channel
5716         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5717         redirect_url = traverse_obj(
5718             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5719         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5720             redirect_url = ''.join((
5721                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5722             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5723             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5724
5725         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5726         if tabs:
5727             selected_tab = self._extract_selected_tab(tabs)
5728             selected_tab_url = urljoin(
5729                 url, traverse_obj(selected_tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
5730             translated_tab_name = selected_tab.get('title', '').lower()
5731
5732             # Prefer tab name from tab url as it is always in en,
5733             # but only when preferred lang is set as it may not extract reliably in all cases.
5734             selected_tab_name = (self._preferred_lang in (None, 'en') and translated_tab_name
5735                                  or selected_tab_url and get_mobj(selected_tab_url)['tab'][1:]  # primary
5736                                  or translated_tab_name)
5737
5738             if selected_tab_name == 'home':
5739                 selected_tab_name = 'featured'
5740             requested_tab_name = mobj['tab'][1:]
5741
5742             if 'no-youtube-channel-redirect' not in compat_opts:
5743                 if requested_tab_name == 'live':  # Live tab should have redirected to the video
5744                     raise UserNotLive(video_id=mobj['id'])
5745                 if requested_tab_name not in ('', selected_tab_name):
5746                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5747                     if not original_tab_name:
5748                         if item_id[:2] == 'UC':
5749                             # Topic channels don't have /videos. Use the equivalent playlist instead
5750                             pl_id = f'UU{item_id[2:]}'
5751                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5752                             try:
5753                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5754                             except ExtractorError:
5755                                 redirect_warning += ' and the playlist redirect gave error'
5756                             else:
5757                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5758                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5759                         if selected_tab_name and selected_tab_name != requested_tab_name:
5760                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5761                     else:
5762                         raise ExtractorError(redirect_warning, expected=True)
5763
5764         if redirect_warning:
5765             self.to_screen(redirect_warning)
5766         self.write_debug(f'Final URL: {url}')
5767
5768         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5769         if 'no-youtube-unavailable-videos' not in compat_opts:
5770             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5771         self._extract_and_report_alerts(data, only_once=True)
5772         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5773         if tabs:
5774             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5775
5776         playlist = traverse_obj(
5777             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5778         if playlist:
5779             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5780
5781         video_id = traverse_obj(
5782             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5783         if video_id:
5784             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5785                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5786             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5787                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5788
5789         raise ExtractorError('Unable to recognize tab page')
5790
5791
5792 class YoutubePlaylistIE(InfoExtractor):
5793     IE_DESC = 'YouTube playlists'
5794     _VALID_URL = r'''(?x)(?:
5795                         (?:https?://)?
5796                         (?:\w+\.)?
5797                         (?:
5798                             (?:
5799                                 youtube(?:kids)?\.com|
5800                                 %(invidious)s
5801                             )
5802                             /.*?\?.*?\blist=
5803                         )?
5804                         (?P<id>%(playlist_id)s)
5805                      )''' % {
5806         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5807         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5808     }
5809     IE_NAME = 'youtube:playlist'
5810     _TESTS = [{
5811         'note': 'issue #673',
5812         'url': 'PLBB231211A4F62143',
5813         'info_dict': {
5814             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5815             'id': 'PLBB231211A4F62143',
5816             'uploader': 'Wickman',
5817             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5818             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5819             'view_count': int,
5820             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5821             'modified_date': r're:\d{8}',
5822             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5823             'channel': 'Wickman',
5824             'tags': [],
5825             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5826         },
5827         'playlist_mincount': 29,
5828     }, {
5829         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5830         'info_dict': {
5831             'title': 'YDL_safe_search',
5832             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5833         },
5834         'playlist_count': 2,
5835         'skip': 'This playlist is private',
5836     }, {
5837         'note': 'embedded',
5838         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5839         'playlist_count': 4,
5840         'info_dict': {
5841             'title': 'JODA15',
5842             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5843             'uploader': 'milan',
5844             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5845             'description': '',
5846             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5847             'tags': [],
5848             'modified_date': '20140919',
5849             'view_count': int,
5850             'channel': 'milan',
5851             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5852             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5853             'availability': 'public',
5854         },
5855         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5856     }, {
5857         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5858         'playlist_mincount': 455,
5859         'info_dict': {
5860             'title': '2018 Chinese New Singles (11/6 updated)',
5861             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5862             'uploader': 'LBK',
5863             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5864             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5865             'channel': 'LBK',
5866             'view_count': int,
5867             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5868             'tags': [],
5869             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5870             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5871             'modified_date': r're:\d{8}',
5872             'availability': 'public',
5873         },
5874         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5875     }, {
5876         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5877         'only_matching': True,
5878     }, {
5879         # music album playlist
5880         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5881         'only_matching': True,
5882     }]
5883
5884     @classmethod
5885     def suitable(cls, url):
5886         if YoutubeTabIE.suitable(url):
5887             return False
5888         from ..utils import parse_qs
5889         qs = parse_qs(url)
5890         if qs.get('v', [None])[0]:
5891             return False
5892         return super().suitable(url)
5893
5894     def _real_extract(self, url):
5895         playlist_id = self._match_id(url)
5896         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5897         url = update_url_query(
5898             'https://www.youtube.com/playlist',
5899             parse_qs(url) or {'list': playlist_id})
5900         if is_music_url:
5901             url = smuggle_url(url, {'is_music_url': True})
5902         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5903
5904
5905 class YoutubeYtBeIE(InfoExtractor):
5906     IE_DESC = 'youtu.be'
5907     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5908     _TESTS = [{
5909         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5910         'info_dict': {
5911             'id': 'yeWKywCrFtk',
5912             'ext': 'mp4',
5913             'title': 'Small Scale Baler and Braiding Rugs',
5914             'uploader': 'Backus-Page House Museum',
5915             'uploader_id': 'backuspagemuseum',
5916             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5917             'upload_date': '20161008',
5918             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5919             'categories': ['Nonprofits & Activism'],
5920             'tags': list,
5921             'like_count': int,
5922             'age_limit': 0,
5923             'playable_in_embed': True,
5924             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5925             'channel': 'Backus-Page House Museum',
5926             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5927             'live_status': 'not_live',
5928             'view_count': int,
5929             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5930             'availability': 'public',
5931             'duration': 59,
5932             'comment_count': int,
5933             'channel_follower_count': int
5934         },
5935         'params': {
5936             'noplaylist': True,
5937             'skip_download': True,
5938         },
5939     }, {
5940         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5941         'only_matching': True,
5942     }]
5943
5944     def _real_extract(self, url):
5945         mobj = self._match_valid_url(url)
5946         video_id = mobj.group('id')
5947         playlist_id = mobj.group('playlist_id')
5948         return self.url_result(
5949             update_url_query('https://www.youtube.com/watch', {
5950                 'v': video_id,
5951                 'list': playlist_id,
5952                 'feature': 'youtu.be',
5953             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5954
5955
5956 class YoutubeLivestreamEmbedIE(InfoExtractor):
5957     IE_DESC = 'YouTube livestream embeds'
5958     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5959     _TESTS = [{
5960         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5961         'only_matching': True,
5962     }]
5963
5964     def _real_extract(self, url):
5965         channel_id = self._match_id(url)
5966         return self.url_result(
5967             f'https://www.youtube.com/channel/{channel_id}/live',
5968             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5969
5970
5971 class YoutubeYtUserIE(InfoExtractor):
5972     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5973     IE_NAME = 'youtube:user'
5974     _VALID_URL = r'ytuser:(?P<id>.+)'
5975     _TESTS = [{
5976         'url': 'ytuser:phihag',
5977         'only_matching': True,
5978     }]
5979
5980     def _real_extract(self, url):
5981         user_id = self._match_id(url)
5982         return self.url_result(
5983             'https://www.youtube.com/user/%s/videos' % user_id,
5984             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5985
5986
5987 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5988     IE_NAME = 'youtube:favorites'
5989     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5990     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5991     _LOGIN_REQUIRED = True
5992     _TESTS = [{
5993         'url': ':ytfav',
5994         'only_matching': True,
5995     }, {
5996         'url': ':ytfavorites',
5997         'only_matching': True,
5998     }]
5999
6000     def _real_extract(self, url):
6001         return self.url_result(
6002             'https://www.youtube.com/playlist?list=LL',
6003             ie=YoutubeTabIE.ie_key())
6004
6005
6006 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6007     IE_NAME = 'youtube:notif'
6008     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6009     _VALID_URL = r':ytnotif(?:ication)?s?'
6010     _LOGIN_REQUIRED = True
6011     _TESTS = [{
6012         'url': ':ytnotif',
6013         'only_matching': True,
6014     }, {
6015         'url': ':ytnotifications',
6016         'only_matching': True,
6017     }]
6018
6019     def _extract_notification_menu(self, response, continuation_list):
6020         notification_list = traverse_obj(
6021             response,
6022             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6023             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6024             expected_type=list) or []
6025         continuation_list[0] = None
6026         for item in notification_list:
6027             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6028             if entry:
6029                 yield entry
6030             continuation = item.get('continuationItemRenderer')
6031             if continuation:
6032                 continuation_list[0] = continuation
6033
6034     def _extract_notification_renderer(self, notification):
6035         video_id = traverse_obj(
6036             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6037         url = f'https://www.youtube.com/watch?v={video_id}'
6038         channel_id = None
6039         if not video_id:
6040             browse_ep = traverse_obj(
6041                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6042             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6043             post_id = self._search_regex(
6044                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6045                 'post id', default=None)
6046             if not channel_id or not post_id:
6047                 return
6048             # The direct /post url redirects to this in the browser
6049             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6050
6051         channel = traverse_obj(
6052             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6053             expected_type=str)
6054         notification_title = self._get_text(notification, 'shortMessage')
6055         if notification_title:
6056             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6057         # TODO: handle recommended videos
6058         title = self._search_regex(
6059             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6060             'video title', default=None)
6061         upload_date = (strftime_or_none(self._parse_time_text(self._get_text(notification, 'sentTimeText')), '%Y%m%d')
6062                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
6063                        else None)
6064         return {
6065             '_type': 'url',
6066             'url': url,
6067             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6068             'video_id': video_id,
6069             'title': title,
6070             'channel_id': channel_id,
6071             'channel': channel,
6072             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6073             'upload_date': upload_date,
6074         }
6075
6076     def _notification_menu_entries(self, ytcfg):
6077         continuation_list = [None]
6078         response = None
6079         for page in itertools.count(1):
6080             ctoken = traverse_obj(
6081                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6082             response = self._extract_response(
6083                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6084                 ep='notification/get_notification_menu', check_get_keys='actions',
6085                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6086             yield from self._extract_notification_menu(response, continuation_list)
6087             if not continuation_list[0]:
6088                 break
6089
6090     def _real_extract(self, url):
6091         display_id = 'notifications'
6092         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6093         self._report_playlist_authcheck(ytcfg)
6094         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6095
6096
6097 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6098     IE_DESC = 'YouTube search'
6099     IE_NAME = 'youtube:search'
6100     _SEARCH_KEY = 'ytsearch'
6101     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6102     _TESTS = [{
6103         'url': 'ytsearch5:youtube-dl test video',
6104         'playlist_count': 5,
6105         'info_dict': {
6106             'id': 'youtube-dl test video',
6107             'title': 'youtube-dl test video',
6108         }
6109     }]
6110
6111
6112 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6113     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6114     _SEARCH_KEY = 'ytsearchdate'
6115     IE_DESC = 'YouTube search, newest videos first'
6116     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6117     _TESTS = [{
6118         'url': 'ytsearchdate5:youtube-dl test video',
6119         'playlist_count': 5,
6120         'info_dict': {
6121             'id': 'youtube-dl test video',
6122             'title': 'youtube-dl test video',
6123         }
6124     }]
6125
6126
6127 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6128     IE_DESC = 'YouTube search URLs with sorting and filter support'
6129     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6130     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6131     _TESTS = [{
6132         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6133         'playlist_mincount': 5,
6134         'info_dict': {
6135             'id': 'youtube-dl test video',
6136             'title': 'youtube-dl test video',
6137         }
6138     }, {
6139         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6140         'playlist_mincount': 5,
6141         'info_dict': {
6142             'id': 'python',
6143             'title': 'python',
6144         }
6145     }, {
6146         'url': 'https://www.youtube.com/results?search_query=%23cats',
6147         'playlist_mincount': 1,
6148         'info_dict': {
6149             'id': '#cats',
6150             'title': '#cats',
6151             # The test suite does not have support for nested playlists
6152             # 'entries': [{
6153             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6154             #     'title': '#cats',
6155             # }],
6156         },
6157     }, {
6158         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6159         'only_matching': True,
6160     }]
6161
6162     def _real_extract(self, url):
6163         qs = parse_qs(url)
6164         query = (qs.get('search_query') or qs.get('q'))[0]
6165         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6166
6167
6168 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6169     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6170     IE_NAME = 'youtube:music:search_url'
6171     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6172     _TESTS = [{
6173         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6174         'playlist_count': 16,
6175         'info_dict': {
6176             'id': 'royalty free music',
6177             'title': 'royalty free music',
6178         }
6179     }, {
6180         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6181         'playlist_mincount': 30,
6182         'info_dict': {
6183             'id': 'royalty free music - songs',
6184             'title': 'royalty free music - songs',
6185         },
6186         'params': {'extract_flat': 'in_playlist'}
6187     }, {
6188         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6189         'playlist_mincount': 30,
6190         'info_dict': {
6191             'id': 'royalty free music - community playlists',
6192             'title': 'royalty free music - community playlists',
6193         },
6194         'params': {'extract_flat': 'in_playlist'}
6195     }]
6196
6197     _SECTIONS = {
6198         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6199         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6200         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6201         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6202         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6203         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6204     }
6205
6206     def _real_extract(self, url):
6207         qs = parse_qs(url)
6208         query = (qs.get('search_query') or qs.get('q'))[0]
6209         params = qs.get('sp', (None,))[0]
6210         if params:
6211             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6212         else:
6213             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6214             params = self._SECTIONS.get(section)
6215             if not params:
6216                 section = None
6217         title = join_nonempty(query, section, delim=' - ')
6218         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6219
6220
6221 class YoutubeFeedsInfoExtractor(InfoExtractor):
6222     """
6223     Base class for feed extractors
6224     Subclasses must re-define the _FEED_NAME property.
6225     """
6226     _LOGIN_REQUIRED = True
6227     _FEED_NAME = 'feeds'
6228
6229     def _real_initialize(self):
6230         YoutubeBaseInfoExtractor._check_login_required(self)
6231
6232     @classproperty
6233     def IE_NAME(self):
6234         return f'youtube:{self._FEED_NAME}'
6235
6236     def _real_extract(self, url):
6237         return self.url_result(
6238             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6239
6240
6241 class YoutubeWatchLaterIE(InfoExtractor):
6242     IE_NAME = 'youtube:watchlater'
6243     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6244     _VALID_URL = r':ytwatchlater'
6245     _TESTS = [{
6246         'url': ':ytwatchlater',
6247         'only_matching': True,
6248     }]
6249
6250     def _real_extract(self, url):
6251         return self.url_result(
6252             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6253
6254
6255 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6256     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6257     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6258     _FEED_NAME = 'recommended'
6259     _LOGIN_REQUIRED = False
6260     _TESTS = [{
6261         'url': ':ytrec',
6262         'only_matching': True,
6263     }, {
6264         'url': ':ytrecommended',
6265         'only_matching': True,
6266     }, {
6267         'url': 'https://youtube.com',
6268         'only_matching': True,
6269     }]
6270
6271
6272 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6273     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6274     _VALID_URL = r':ytsub(?:scription)?s?'
6275     _FEED_NAME = 'subscriptions'
6276     _TESTS = [{
6277         'url': ':ytsubs',
6278         'only_matching': True,
6279     }, {
6280         'url': ':ytsubscriptions',
6281         'only_matching': True,
6282     }]
6283
6284
6285 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6286     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6287     _VALID_URL = r':ythis(?:tory)?'
6288     _FEED_NAME = 'history'
6289     _TESTS = [{
6290         'url': ':ythistory',
6291         'only_matching': True,
6292     }]
6293
6294
6295 class YoutubeStoriesIE(InfoExtractor):
6296     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6297     IE_NAME = 'youtube:stories'
6298     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6299     _TESTS = [{
6300         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6301         'only_matching': True,
6302     }]
6303
6304     def _real_extract(self, url):
6305         playlist_id = f'RLTD{self._match_id(url)}'
6306         return self.url_result(
6307             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6308             ie=YoutubeTabIE, video_id=playlist_id)
6309
6310
6311 class YoutubeShortsAudioPivotIE(InfoExtractor):
6312     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6313     IE_NAME = 'youtube:shorts:pivot:audio'
6314     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6315     _TESTS = [{
6316         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6317         'only_matching': True,
6318     }]
6319
6320     @staticmethod
6321     def _generate_audio_pivot_params(video_id):
6322         """
6323         Generates sfv_audio_pivot browse params for this video id
6324         """
6325         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6326         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6327
6328     def _real_extract(self, url):
6329         video_id = self._match_id(url)
6330         return self.url_result(
6331             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6332             ie=YoutubeTabIE)
6333
6334
6335 class YoutubeTruncatedURLIE(InfoExtractor):
6336     IE_NAME = 'youtube:truncated_url'
6337     IE_DESC = False  # Do not list
6338     _VALID_URL = r'''(?x)
6339         (?:https?://)?
6340         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6341         (?:watch\?(?:
6342             feature=[a-z_]+|
6343             annotation_id=annotation_[^&]+|
6344             x-yt-cl=[0-9]+|
6345             hl=[^&]*|
6346             t=[0-9]+
6347         )?
6348         |
6349             attribution_link\?a=[^&]+
6350         )
6351         $
6352     '''
6353
6354     _TESTS = [{
6355         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6356         'only_matching': True,
6357     }, {
6358         'url': 'https://www.youtube.com/watch?',
6359         'only_matching': True,
6360     }, {
6361         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6362         'only_matching': True,
6363     }, {
6364         'url': 'https://www.youtube.com/watch?feature=foo',
6365         'only_matching': True,
6366     }, {
6367         'url': 'https://www.youtube.com/watch?hl=en-GB',
6368         'only_matching': True,
6369     }, {
6370         'url': 'https://www.youtube.com/watch?t=2372',
6371         'only_matching': True,
6372     }]
6373
6374     def _real_extract(self, url):
6375         raise ExtractorError(
6376             'Did you forget to quote the URL? Remember that & is a meta '
6377             'character in most shells, so you want to put the URL in quotes, '
6378             'like  youtube-dl '
6379             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6380             ' or simply  youtube-dl BaW_jenozKc  .',
6381             expected=True)
6382
6383
6384 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6385     IE_NAME = 'youtube:clip'
6386     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6387     _TESTS = [{
6388         # FIXME: Other metadata should be extracted from the clip, not from the base video
6389         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6390         'info_dict': {
6391             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6392             'ext': 'mp4',
6393             'section_start': 29.0,
6394             'section_end': 39.7,
6395             'duration': 10.7,
6396             'age_limit': 0,
6397             'availability': 'public',
6398             'categories': ['Gaming'],
6399             'channel': 'Scott The Woz',
6400             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6401             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6402             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6403             'like_count': int,
6404             'playable_in_embed': True,
6405             'tags': 'count:17',
6406             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6407             'title': 'Mobile Games on Console - Scott The Woz',
6408             'upload_date': '20210920',
6409             'uploader': 'Scott The Woz',
6410             'uploader_id': 'scottthewoz',
6411             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6412             'view_count': int,
6413             'live_status': 'not_live',
6414             'channel_follower_count': int
6415         }
6416     }]
6417
6418     def _real_extract(self, url):
6419         clip_id = self._match_id(url)
6420         _, data = self._extract_webpage(url, clip_id)
6421
6422         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6423         if not video_id:
6424             raise ExtractorError('Unable to find video ID')
6425
6426         clip_data = traverse_obj(data, (
6427             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6428             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6429             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6430             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6431
6432         return {
6433             '_type': 'url_transparent',
6434             'url': f'https://www.youtube.com/watch?v={video_id}',
6435             'ie_key': YoutubeIE.ie_key(),
6436             'id': clip_id,
6437             'section_start': int(clip_data['startTimeMs']) / 1000,
6438             'section_end': int(clip_data['endTimeMs']) / 1000,
6439         }
6440
6441
6442 class YoutubeTruncatedIDIE(InfoExtractor):
6443     IE_NAME = 'youtube:truncated_id'
6444     IE_DESC = False  # Do not list
6445     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6446
6447     _TESTS = [{
6448         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6449         'only_matching': True,
6450     }]
6451
6452     def _real_extract(self, url):
6453         video_id = self._match_id(url)
6454         raise ExtractorError(
6455             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6456             expected=True)