yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import collections
   4 import copy
   5 import datetime
   6 import enum
   7 import hashlib
   8 import itertools
   9 import json
  10 import math
  11 import os.path
  12 import random
  13 import re
  14 import sys
  15 import threading
  16 import time
  17 import traceback
  18 import urllib.error
  19 import urllib.parse
  20
  21 from .common import InfoExtractor, SearchInfoExtractor
  22 from .openload import PhantomJSwrapper
  23 from ..compat import functools
  24 from ..jsinterp import JSInterpreter
  25 from ..utils import (
  26     NO_DEFAULT,
  27     ExtractorError,
  28     LazyList,
  29     UserNotLive,
  30     bug_reports_message,
  31     classproperty,
  32     clean_html,
  33     datetime_from_str,
  34     dict_get,
  35     filter_dict,
  36     float_or_none,
  37     format_field,
  38     get_first,
  39     int_or_none,
  40     is_html,
  41     join_nonempty,
  42     js_to_json,
  43     mimetype2ext,
  44     network_exceptions,
  45     orderedSet,
  46     parse_codecs,
  47     parse_count,
  48     parse_duration,
  49     parse_iso8601,
  50     parse_qs,
  51     qualities,
  52     remove_start,
  53     smuggle_url,
  54     str_or_none,
  55     str_to_int,
  56     strftime_or_none,
  57     traverse_obj,
  58     try_get,
  59     unescapeHTML,
  60     unified_strdate,
  61     unified_timestamp,
  62     unsmuggle_url,
  63     update_url_query,
  64     url_or_none,
  65     urljoin,
  66     variadic,
  67 )
  68
  69 # any clients starting with _ cannot be explicitly requested by the user
  70 INNERTUBE_CLIENTS = {
  71     'web': {
  72         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  73         'INNERTUBE_CONTEXT': {
  74             'client': {
  75                 'clientName': 'WEB',
  76                 'clientVersion': '2.20220801.00.00',
  77             }
  78         },
  79         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  80     },
  81     'web_embedded': {
  82         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  83         'INNERTUBE_CONTEXT': {
  84             'client': {
  85                 'clientName': 'WEB_EMBEDDED_PLAYER',
  86                 'clientVersion': '1.20220731.00.00',
  87             },
  88         },
  89         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  90     },
  91     'web_music': {
  92         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  93         'INNERTUBE_HOST': 'music.youtube.com',
  94         'INNERTUBE_CONTEXT': {
  95             'client': {
  96                 'clientName': 'WEB_REMIX',
  97                 'clientVersion': '1.20220727.01.00',
  98             }
  99         },
 100         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 101     },
 102     'web_creator': {
 103         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 104         'INNERTUBE_CONTEXT': {
 105             'client': {
 106                 'clientName': 'WEB_CREATOR',
 107                 'clientVersion': '1.20220726.00.00',
 108             }
 109         },
 110         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 111     },
 112     'android': {
 113         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 114         'INNERTUBE_CONTEXT': {
 115             'client': {
 116                 'clientName': 'ANDROID',
 117                 'clientVersion': '17.31.35',
 118                 'androidSdkVersion': 30,
 119                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 120             }
 121         },
 122         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 123         'REQUIRE_JS_PLAYER': False
 124     },
 125     'android_embedded': {
 126         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 127         'INNERTUBE_CONTEXT': {
 128             'client': {
 129                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 130                 'clientVersion': '17.31.35',
 131                 'androidSdkVersion': 30,
 132                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 133             },
 134         },
 135         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 136         'REQUIRE_JS_PLAYER': False
 137     },
 138     'android_music': {
 139         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 140         'INNERTUBE_CONTEXT': {
 141             'client': {
 142                 'clientName': 'ANDROID_MUSIC',
 143                 'clientVersion': '5.16.51',
 144                 'androidSdkVersion': 30,
 145                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 146             }
 147         },
 148         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 149         'REQUIRE_JS_PLAYER': False
 150     },
 151     'android_creator': {
 152         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 153         'INNERTUBE_CONTEXT': {
 154             'client': {
 155                 'clientName': 'ANDROID_CREATOR',
 156                 'clientVersion': '22.30.100',
 157                 'androidSdkVersion': 30,
 158                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 159             },
 160         },
 161         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 162         'REQUIRE_JS_PLAYER': False
 163     },
 164     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 165     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 166     'ios': {
 167         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 168         'INNERTUBE_CONTEXT': {
 169             'client': {
 170                 'clientName': 'IOS',
 171                 'clientVersion': '17.33.2',
 172                 'deviceModel': 'iPhone14,3',
 173                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 174             }
 175         },
 176         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 177         'REQUIRE_JS_PLAYER': False
 178     },
 179     'ios_embedded': {
 180         'INNERTUBE_CONTEXT': {
 181             'client': {
 182                 'clientName': 'IOS_MESSAGES_EXTENSION',
 183                 'clientVersion': '17.33.2',
 184                 'deviceModel': 'iPhone14,3',
 185                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 186             },
 187         },
 188         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 189         'REQUIRE_JS_PLAYER': False
 190     },
 191     'ios_music': {
 192         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 193         'INNERTUBE_CONTEXT': {
 194             'client': {
 195                 'clientName': 'IOS_MUSIC',
 196                 'clientVersion': '5.21',
 197                 'deviceModel': 'iPhone14,3',
 198                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 199             },
 200         },
 201         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 202         'REQUIRE_JS_PLAYER': False
 203     },
 204     'ios_creator': {
 205         'INNERTUBE_CONTEXT': {
 206             'client': {
 207                 'clientName': 'IOS_CREATOR',
 208                 'clientVersion': '22.33.101',
 209                 'deviceModel': 'iPhone14,3',
 210                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 211             },
 212         },
 213         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 214         'REQUIRE_JS_PLAYER': False
 215     },
 216     # mweb has 'ultralow' formats
 217     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 218     'mweb': {
 219         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 220         'INNERTUBE_CONTEXT': {
 221             'client': {
 222                 'clientName': 'MWEB',
 223                 'clientVersion': '2.20220801.00.00',
 224             }
 225         },
 226         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 227     },
 228     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 229     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 230     'tv_embedded': {
 231         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 232         'INNERTUBE_CONTEXT': {
 233             'client': {
 234                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 235                 'clientVersion': '2.0',
 236             },
 237         },
 238         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 239     },
 240 }
 241
 242
 243 def _split_innertube_client(client_name):
 244     variant, *base = client_name.rsplit('.', 1)
 245     if base:
 246         return variant, base[0], variant
 247     base, *variant = client_name.split('_', 1)
 248     return client_name, base, variant[0] if variant else None
 249
 250
 251 def build_innertube_clients():
 252     THIRD_PARTY = {
 253         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 254     }
 255     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 256     priority = qualities(BASE_CLIENTS[::-1])
 257
 258     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 259         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 260         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 261         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 262         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 263
 264         _, base_client, variant = _split_innertube_client(client)
 265         ytcfg['priority'] = 10 * priority(base_client)
 266
 267         if not variant:
 268             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 269             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 270             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 271             embedscreen['priority'] -= 3
 272         elif variant == 'embedded':
 273             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 274             ytcfg['priority'] -= 2
 275         else:
 276             ytcfg['priority'] -= 3
 277
 278
 279 build_innertube_clients()
 280
 281
 282 class BadgeType(enum.Enum):
 283     AVAILABILITY_UNLISTED = enum.auto()
 284     AVAILABILITY_PRIVATE = enum.auto()
 285     AVAILABILITY_PUBLIC = enum.auto()
 286     AVAILABILITY_PREMIUM = enum.auto()
 287     AVAILABILITY_SUBSCRIPTION = enum.auto()
 288     LIVE_NOW = enum.auto()
 289
 290
 291 class YoutubeBaseInfoExtractor(InfoExtractor):
 292     """Provide base functions for Youtube extractors"""
 293
 294     _RESERVED_NAMES = (
 295         r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
 296         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 297         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 298         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 299
 300     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 301
 302     # _NETRC_MACHINE = 'youtube'
 303
 304     # If True it will raise an error if no login info is provided
 305     _LOGIN_REQUIRED = False
 306
 307     _INVIDIOUS_SITES = (
 308         # invidious-redirect websites
 309         r'(?:www\.)?redirect\.invidious\.io',
 310         r'(?:(?:www|dev)\.)?invidio\.us',
 311         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 312         r'(?:www\.)?invidious\.pussthecat\.org',
 313         r'(?:www\.)?invidious\.zee\.li',
 314         r'(?:www\.)?invidious\.ethibox\.fr',
 315         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 316         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 317         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 318         # youtube-dl invidious instances list
 319         r'(?:(?:www|no)\.)?invidiou\.sh',
 320         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 321         r'(?:www\.)?invidious\.kabi\.tk',
 322         r'(?:www\.)?invidious\.mastodon\.host',
 323         r'(?:www\.)?invidious\.zapashcanon\.fr',
 324         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 325         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 326         r'(?:www\.)?invidious\.himiko\.cloud',
 327         r'(?:www\.)?invidious\.reallyancient\.tech',
 328         r'(?:www\.)?invidious\.tube',
 329         r'(?:www\.)?invidiou\.site',
 330         r'(?:www\.)?invidious\.site',
 331         r'(?:www\.)?invidious\.xyz',
 332         r'(?:www\.)?invidious\.nixnet\.xyz',
 333         r'(?:www\.)?invidious\.048596\.xyz',
 334         r'(?:www\.)?invidious\.drycat\.fr',
 335         r'(?:www\.)?inv\.skyn3t\.in',
 336         r'(?:www\.)?tube\.poal\.co',
 337         r'(?:www\.)?tube\.connect\.cafe',
 338         r'(?:www\.)?vid\.wxzm\.sx',
 339         r'(?:www\.)?vid\.mint\.lgbt',
 340         r'(?:www\.)?vid\.puffyan\.us',
 341         r'(?:www\.)?yewtu\.be',
 342         r'(?:www\.)?yt\.elukerio\.org',
 343         r'(?:www\.)?yt\.lelux\.fi',
 344         r'(?:www\.)?invidious\.ggc-project\.de',
 345         r'(?:www\.)?yt\.maisputain\.ovh',
 346         r'(?:www\.)?ytprivate\.com',
 347         r'(?:www\.)?invidious\.13ad\.de',
 348         r'(?:www\.)?invidious\.toot\.koeln',
 349         r'(?:www\.)?invidious\.fdn\.fr',
 350         r'(?:www\.)?watch\.nettohikari\.com',
 351         r'(?:www\.)?invidious\.namazso\.eu',
 352         r'(?:www\.)?invidious\.silkky\.cloud',
 353         r'(?:www\.)?invidious\.exonip\.de',
 354         r'(?:www\.)?invidious\.riverside\.rocks',
 355         r'(?:www\.)?invidious\.blamefran\.net',
 356         r'(?:www\.)?invidious\.moomoo\.de',
 357         r'(?:www\.)?ytb\.trom\.tf',
 358         r'(?:www\.)?yt\.cyberhost\.uk',
 359         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 360         r'(?:www\.)?qklhadlycap4cnod\.onion',
 361         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 362         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 363         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 364         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 365         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 366         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 367         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 368         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 369         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 370         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 371         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 372         r'(?:www\.)?piped\.kavin\.rocks',
 373         r'(?:www\.)?piped\.tokhmi\.xyz',
 374         r'(?:www\.)?piped\.syncpundit\.io',
 375         r'(?:www\.)?piped\.mha\.fi',
 376         r'(?:www\.)?watch\.whatever\.social',
 377         r'(?:www\.)?piped\.garudalinux\.org',
 378         r'(?:www\.)?piped\.rivo\.lol',
 379         r'(?:www\.)?piped-libre\.kavin\.rocks',
 380         r'(?:www\.)?yt\.jae\.fi',
 381         r'(?:www\.)?piped\.mint\.lgbt',
 382         r'(?:www\.)?il\.ax',
 383         r'(?:www\.)?piped\.esmailelbob\.xyz',
 384         r'(?:www\.)?piped\.projectsegfau\.lt',
 385         r'(?:www\.)?piped\.privacydev\.net',
 386         r'(?:www\.)?piped\.palveluntarjoaja\.eu',
 387         r'(?:www\.)?piped\.smnz\.de',
 388         r'(?:www\.)?piped\.adminforge\.de',
 389         r'(?:www\.)?watch\.whatevertinfoil\.de',
 390         r'(?:www\.)?piped\.qdi\.fi',
 391         r'(?:www\.)?piped\.video',
 392         r'(?:www\.)?piped\.aeong\.one',
 393     )
 394
 395     # extracted from account/account_menu ep
 396     # XXX: These are the supported YouTube UI and API languages,
 397     # which is slightly different from languages supported for translation in YouTube studio
 398     _SUPPORTED_LANG_CODES = [
 399         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 400         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 401         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 402         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 403         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 404         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 405     ]
 406
 407     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 408
 409     @functools.cached_property
 410     def _preferred_lang(self):
 411         """
 412         Returns a language code supported by YouTube for the user preferred language.
 413         Returns None if no preferred language set.
 414         """
 415         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 416         if not preferred_lang:
 417             return
 418         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 419             raise ExtractorError(
 420                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 421                 expected=True)
 422         elif preferred_lang != 'en':
 423             self.report_warning(
 424                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 425         return preferred_lang
 426
 427     def _initialize_consent(self):
 428         cookies = self._get_cookies('https://www.youtube.com/')
 429         if cookies.get('__Secure-3PSID'):
 430             return
 431         consent_id = None
 432         consent = cookies.get('CONSENT')
 433         if consent:
 434             if 'YES' in consent.value:
 435                 return
 436             consent_id = self._search_regex(
 437                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 438         if not consent_id:
 439             consent_id = random.randint(100, 999)
 440         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 441
 442     def _initialize_pref(self):
 443         cookies = self._get_cookies('https://www.youtube.com/')
 444         pref_cookie = cookies.get('PREF')
 445         pref = {}
 446         if pref_cookie:
 447             try:
 448                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 449             except ValueError:
 450                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 451         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 452         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 453
 454     def _real_initialize(self):
 455         self._initialize_pref()
 456         self._initialize_consent()
 457         self._check_login_required()
 458
 459     def _check_login_required(self):
 460         if self._LOGIN_REQUIRED and not self._cookies_passed:
 461             self.raise_login_required('Login details are needed to download this content', method='cookies')
 462
 463     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 464     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 465
 466     def _get_default_ytcfg(self, client='web'):
 467         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 468
 469     def _get_innertube_host(self, client='web'):
 470         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 471
 472     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 473         # try_get but with fallback to default ytcfg client values when present
 474         _func = lambda y: try_get(y, getter, expected_type)
 475         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 476
 477     def _extract_client_name(self, ytcfg, default_client='web'):
 478         return self._ytcfg_get_safe(
 479             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 480                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 481
 482     def _extract_client_version(self, ytcfg, default_client='web'):
 483         return self._ytcfg_get_safe(
 484             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 485                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 486
 487     def _select_api_hostname(self, req_api_hostname, default_client=None):
 488         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 489                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 490
 491     def _extract_api_key(self, ytcfg=None, default_client='web'):
 492         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 493
 494     def _extract_context(self, ytcfg=None, default_client='web'):
 495         context = get_first(
 496             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 497         # Enforce language and tz for extraction
 498         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 499         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 500         return context
 501
 502     _SAPISID = None
 503
 504     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 505         time_now = round(time.time())
 506         if self._SAPISID is None:
 507             yt_cookies = self._get_cookies('https://www.youtube.com')
 508             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 509             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 510             sapisid_cookie = dict_get(
 511                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 512             if sapisid_cookie and sapisid_cookie.value:
 513                 self._SAPISID = sapisid_cookie.value
 514                 self.write_debug('Extracted SAPISID cookie')
 515                 # SAPISID cookie is required if not already present
 516                 if not yt_cookies.get('SAPISID'):
 517                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 518                     self._set_cookie(
 519                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 520             else:
 521                 self._SAPISID = False
 522         if not self._SAPISID:
 523             return None
 524         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 525         sapisidhash = hashlib.sha1(
 526             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 527         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 528
 529     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 530                   note='Downloading API JSON', errnote='Unable to download API page',
 531                   context=None, api_key=None, api_hostname=None, default_client='web'):
 532
 533         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 534         data.update(query)
 535         real_headers = self.generate_api_headers(default_client=default_client)
 536         real_headers.update({'content-type': 'application/json'})
 537         if headers:
 538             real_headers.update(headers)
 539         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 540                    or api_key or self._extract_api_key(default_client=default_client))
 541         return self._download_json(
 542             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 543             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 544             data=json.dumps(data).encode('utf8'), headers=real_headers,
 545             query={'key': api_key, 'prettyPrint': 'false'})
 546
 547     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 548         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 549
 550     @staticmethod
 551     def _extract_session_index(*data):
 552         """
 553         Index of current account in account list.
 554         See: https://github.com/yt-dlp/yt-dlp/pull/519
 555         """
 556         for ytcfg in data:
 557             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 558             if session_index is not None:
 559                 return session_index
 560
 561     # Deprecated?
 562     def _extract_identity_token(self, ytcfg=None, webpage=None):
 563         if ytcfg:
 564             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 565             if token:
 566                 return token
 567         if webpage:
 568             return self._search_regex(
 569                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 570                 'identity token', default=None, fatal=False)
 571
 572     @staticmethod
 573     def _extract_account_syncid(*args):
 574         """
 575         Extract syncId required to download private playlists of secondary channels
 576         @params response and/or ytcfg
 577         """
 578         for data in args:
 579             # ytcfg includes channel_syncid if on secondary channel
 580             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 581             if delegated_sid:
 582                 return delegated_sid
 583             sync_ids = (try_get(
 584                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 585                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 586             if len(sync_ids) >= 2 and sync_ids[1]:
 587                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 588                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 589                 return sync_ids[0]
 590
 591     @staticmethod
 592     def _extract_visitor_data(*args):
 593         """
 594         Extracts visitorData from an API response or ytcfg
 595         Appears to be used to track session state
 596         """
 597         return get_first(
 598             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 599             expected_type=str)
 600
 601     @functools.cached_property
 602     def is_authenticated(self):
 603         return bool(self._generate_sapisidhash_header())
 604
 605     def extract_ytcfg(self, video_id, webpage):
 606         if not webpage:
 607             return {}
 608         return self._parse_json(
 609             self._search_regex(
 610                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 611                 default='{}'), video_id, fatal=False) or {}
 612
 613     def generate_api_headers(
 614             self, *, ytcfg=None, account_syncid=None, session_index=None,
 615             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 616
 617         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 618         headers = {
 619             'X-YouTube-Client-Name': str(
 620                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 621             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 622             'Origin': origin,
 623             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 624             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 625             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 626             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 627         }
 628         if session_index is None:
 629             session_index = self._extract_session_index(ytcfg)
 630         if account_syncid or session_index is not None:
 631             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 632
 633         auth = self._generate_sapisidhash_header(origin)
 634         if auth is not None:
 635             headers['Authorization'] = auth
 636             headers['X-Origin'] = origin
 637         return filter_dict(headers)
 638
 639     def _download_ytcfg(self, client, video_id):
 640         url = {
 641             'web': 'https://www.youtube.com',
 642             'web_music': 'https://music.youtube.com',
 643             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 644         }.get(client)
 645         if not url:
 646             return {}
 647         webpage = self._download_webpage(
 648             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 649         return self.extract_ytcfg(video_id, webpage) or {}
 650
 651     @staticmethod
 652     def _build_api_continuation_query(continuation, ctp=None):
 653         query = {
 654             'continuation': continuation
 655         }
 656         # TODO: Inconsistency with clickTrackingParams.
 657         # Currently we have a fixed ctp contained within context (from ytcfg)
 658         # and a ctp in root query for continuation.
 659         if ctp:
 660             query['clickTracking'] = {'clickTrackingParams': ctp}
 661         return query
 662
 663     @classmethod
 664     def _extract_next_continuation_data(cls, renderer):
 665         next_continuation = try_get(
 666             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 667                        lambda x: x['continuation']['reloadContinuationData']), dict)
 668         if not next_continuation:
 669             return
 670         continuation = next_continuation.get('continuation')
 671         if not continuation:
 672             return
 673         ctp = next_continuation.get('clickTrackingParams')
 674         return cls._build_api_continuation_query(continuation, ctp)
 675
 676     @classmethod
 677     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 678         if isinstance(continuation_ep, dict):
 679             continuation = try_get(
 680                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 681             if not continuation:
 682                 return
 683             ctp = continuation_ep.get('clickTrackingParams')
 684             return cls._build_api_continuation_query(continuation, ctp)
 685
 686     @classmethod
 687     def _extract_continuation(cls, renderer):
 688         next_continuation = cls._extract_next_continuation_data(renderer)
 689         if next_continuation:
 690             return next_continuation
 691
 692         return traverse_obj(renderer, (
 693             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 694             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 695         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 696
 697     @classmethod
 698     def _extract_alerts(cls, data):
 699         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 700             if not isinstance(alert_dict, dict):
 701                 continue
 702             for alert in alert_dict.values():
 703                 alert_type = alert.get('type')
 704                 if not alert_type:
 705                     continue
 706                 message = cls._get_text(alert, 'text')
 707                 if message:
 708                     yield alert_type, message
 709
 710     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 711         errors, warnings = [], []
 712         for alert_type, alert_message in alerts:
 713             if alert_type.lower() == 'error' and fatal:
 714                 errors.append([alert_type, alert_message])
 715             elif alert_message not in self._IGNORED_WARNINGS:
 716                 warnings.append([alert_type, alert_message])
 717
 718         for alert_type, alert_message in (warnings + errors[:-1]):
 719             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 720         if errors:
 721             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 722
 723     def _extract_and_report_alerts(self, data, *args, **kwargs):
 724         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 725
 726     def _extract_badges(self, renderer: dict):
 727         privacy_icon_map = {
 728             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 729             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 730             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 731         }
 732
 733         badge_style_map = {
 734             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 735             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 736             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 737         }
 738
 739         label_map = {
 740             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 741             'private': BadgeType.AVAILABILITY_PRIVATE,
 742             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 743             'live': BadgeType.LIVE_NOW,
 744             'premium': BadgeType.AVAILABILITY_PREMIUM
 745         }
 746
 747         badges = []
 748         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer'), default=[]):
 749             badge_type = (
 750                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 751                 or badge_style_map.get(traverse_obj(badge, 'style'))
 752             )
 753             if badge_type:
 754                 badges.append({'type': badge_type})
 755                 continue
 756
 757             # fallback, won't work in some languages
 758             label = traverse_obj(badge, 'label', expected_type=str, default='')
 759             for match, label_badge_type in label_map.items():
 760                 if match in label.lower():
 761                     badges.append({'type': badge_type})
 762                     continue
 763
 764         return badges
 765
 766     @staticmethod
 767     def _has_badge(badges, badge_type):
 768         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 769
 770     @staticmethod
 771     def _get_text(data, *path_list, max_runs=None):
 772         for path in path_list or [None]:
 773             if path is None:
 774                 obj = [data]
 775             else:
 776                 obj = traverse_obj(data, path, default=[])
 777                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 778                     obj = [obj]
 779             for item in obj:
 780                 text = try_get(item, lambda x: x['simpleText'], str)
 781                 if text:
 782                     return text
 783                 runs = try_get(item, lambda x: x['runs'], list) or []
 784                 if not runs and isinstance(item, list):
 785                     runs = item
 786
 787                 runs = runs[:min(len(runs), max_runs or len(runs))]
 788                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 789                 if text:
 790                     return text
 791
 792     def _get_count(self, data, *path_list):
 793         count_text = self._get_text(data, *path_list) or ''
 794         count = parse_count(count_text)
 795         if count is None:
 796             count = str_to_int(
 797                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 798         return count
 799
 800     @staticmethod
 801     def _extract_thumbnails(data, *path_list):
 802         """
 803         Extract thumbnails from thumbnails dict
 804         @param path_list: path list to level that contains 'thumbnails' key
 805         """
 806         thumbnails = []
 807         for path in path_list or [()]:
 808             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 809                 thumbnail_url = url_or_none(thumbnail.get('url'))
 810                 if not thumbnail_url:
 811                     continue
 812                 # Sometimes youtube gives a wrong thumbnail URL. See:
 813                 # https://github.com/yt-dlp/yt-dlp/issues/233
 814                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 815                 if 'maxresdefault' in thumbnail_url:
 816                     thumbnail_url = thumbnail_url.split('?')[0]
 817                 thumbnails.append({
 818                     'url': thumbnail_url,
 819                     'height': int_or_none(thumbnail.get('height')),
 820                     'width': int_or_none(thumbnail.get('width')),
 821                 })
 822         return thumbnails
 823
 824     @staticmethod
 825     def extract_relative_time(relative_time_text):
 826         """
 827         Extracts a relative time from string and converts to dt object
 828         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 829         """
 830         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 831         if mobj:
 832             start = mobj.group('start')
 833             if start:
 834                 return datetime_from_str(start)
 835             try:
 836                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 837             except ValueError:
 838                 return None
 839
 840     def _parse_time_text(self, text):
 841         if not text:
 842             return
 843         dt = self.extract_relative_time(text)
 844         timestamp = None
 845         if isinstance(dt, datetime.datetime):
 846             timestamp = calendar.timegm(dt.timetuple())
 847
 848         if timestamp is None:
 849             timestamp = (
 850                 unified_timestamp(text) or unified_timestamp(
 851                     self._search_regex(
 852                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 853                         text.lower(), 'time text', default=None)))
 854
 855         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 856             self.report_warning(
 857                 f'Cannot parse localized time text "{text}"', only_once=True)
 858         return timestamp
 859
 860     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 861                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 862                           default_client='web'):
 863         for retry in self.RetryManager():
 864             try:
 865                 response = self._call_api(
 866                     ep=ep, fatal=True, headers=headers,
 867                     video_id=item_id, query=query, note=note,
 868                     context=self._extract_context(ytcfg, default_client),
 869                     api_key=self._extract_api_key(ytcfg, default_client),
 870                     api_hostname=api_hostname, default_client=default_client)
 871             except ExtractorError as e:
 872                 if not isinstance(e.cause, network_exceptions):
 873                     return self._error_or_warning(e, fatal=fatal)
 874                 elif not isinstance(e.cause, urllib.error.HTTPError):
 875                     retry.error = e
 876                     continue
 877
 878                 first_bytes = e.cause.read(512)
 879                 if not is_html(first_bytes):
 880                     yt_error = try_get(
 881                         self._parse_json(
 882                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 883                         lambda x: x['error']['message'], str)
 884                     if yt_error:
 885                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 886                 # Downloading page may result in intermittent 5xx HTTP error
 887                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 888                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 889                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 890                 if e.cause.code not in (403, 429):
 891                     retry.error = e
 892                     continue
 893                 return self._error_or_warning(e, fatal=fatal)
 894
 895             try:
 896                 self._extract_and_report_alerts(response, only_once=True)
 897             except ExtractorError as e:
 898                 # YouTube servers may return errors we want to retry on in a 200 OK response
 899                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 900                 if 'unknown error' in e.msg.lower():
 901                     retry.error = e
 902                     continue
 903                 return self._error_or_warning(e, fatal=fatal)
 904             # Youtube sometimes sends incomplete data
 905             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 906             if not traverse_obj(response, *variadic(check_get_keys)):
 907                 retry.error = ExtractorError('Incomplete data received', expected=True)
 908                 continue
 909
 910             return response
 911
 912     @staticmethod
 913     def is_music_url(url):
 914         return re.match(r'https?://music\.youtube\.com/', url) is not None
 915
 916     def _extract_video(self, renderer):
 917         video_id = renderer.get('videoId')
 918
 919         reel_header_renderer = traverse_obj(renderer, (
 920             'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
 921             'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
 922
 923         title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
 924         description = self._get_text(renderer, 'descriptionSnippet')
 925
 926         duration = int_or_none(renderer.get('lengthSeconds'))
 927         if duration is None:
 928             duration = parse_duration(self._get_text(
 929                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 930         if duration is None:
 931             # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
 932             duration = parse_duration(self._search_regex(
 933                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 934                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 935                 video_id, default=None, group='duration'))
 936
 937         channel_id = traverse_obj(
 938             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 939             expected_type=str, get_all=False)
 940         if not channel_id:
 941             channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
 942
 943         overlay_style = traverse_obj(
 944             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 945             get_all=False, expected_type=str)
 946         badges = self._extract_badges(renderer)
 947
 948         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 949             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 950             expected_type=str)) or ''
 951         url = f'https://www.youtube.com/watch?v={video_id}'
 952         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 953             url = f'https://www.youtube.com/shorts/{video_id}'
 954
 955         time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
 956                      or self._get_text(reel_header_renderer, 'timestampText') or '')
 957         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 958
 959         live_status = (
 960             'is_upcoming' if scheduled_timestamp is not None
 961             else 'was_live' if 'streamed' in time_text.lower()
 962             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
 963             else None)
 964
 965         # videoInfo is a string like '50K views • 10 years ago'.
 966         view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
 967         view_count = (0 if 'no views' in view_count_text.lower()
 968                       else self._get_count({'simpleText': view_count_text}))
 969         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
 970
 971         return {
 972             '_type': 'url',
 973             'ie_key': YoutubeIE.ie_key(),
 974             'id': video_id,
 975             'url': url,
 976             'title': title,
 977             'description': description,
 978             'duration': duration,
 979             'channel_id': channel_id,
 980             'channel': (self._get_text(renderer, 'ownerText', 'shortBylineText')
 981                         or self._get_text(reel_header_renderer, 'channelTitleText')),
 982             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
 983             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
 984             'timestamp': (self._parse_time_text(time_text)
 985                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
 986                           else None),
 987             'release_timestamp': scheduled_timestamp,
 988             'availability':
 989                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
 990                 else self._availability(
 991                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
 992                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
 993                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
 994                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
 995             view_count_field: view_count,
 996             'live_status': live_status
 997         }
 998
 999
1000 class YoutubeIE(YoutubeBaseInfoExtractor):
1001     IE_DESC = 'YouTube'
1002     _VALID_URL = r"""(?x)^
1003                      (
1004                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
1005                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
1006                             (?:www\.)?deturl\.com/www\.youtube\.com|
1007                             (?:www\.)?pwnyoutube\.com|
1008                             (?:www\.)?hooktube\.com|
1009                             (?:www\.)?yourepeat\.com|
1010                             tube\.majestyc\.net|
1011                             %(invidious)s|
1012                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
1013                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
1014                          (?:                                                  # the various things that can precede the ID:
1015                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
1016                              |(?:                                             # or the v= param in all its forms
1017                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
1018                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
1019                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
1020                                  v=
1021                              )
1022                          ))
1023                          |(?:
1024                             youtu\.be|                                        # just youtu.be/xxxx
1025                             vid\.plus|                                        # or vid.plus/xxxx
1026                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
1027                             %(invidious)s
1028                          )/
1029                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1030                          )
1031                      )?                                                       # all until now is optional -> you can pass the naked ID
1032                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1033                      (?(1).+)?                                                # if we found the ID, everything can follow
1034                      (?:\#|$)""" % {
1035         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1036     }
1037     _EMBED_REGEX = [
1038         r'''(?x)
1039             (?:
1040                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1041                 data-video-url=|
1042                 <embed[^>]+?src=|
1043                 embedSWF\(?:\s*|
1044                 <object[^>]+data=|
1045                 new\s+SWFObject\(
1046             )
1047             (["\'])
1048                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1049                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1050             \1''',
1051         # https://wordpress.org/plugins/lazy-load-for-videos/
1052         r'''(?xs)
1053             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1054             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1055     ]
1056     _RETURN_TYPE = 'video'  # XXX: How to handle multifeed?
1057
1058     _PLAYER_INFO_RE = (
1059         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1060         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1061         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1062     )
1063     _formats = {
1064         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1065         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1066         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1067         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1068         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1069         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1070         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1071         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1072         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1073         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1074         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1075         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1076         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1077         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1078         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1079         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1080         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1081         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1082
1083
1084         # 3D videos
1085         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1086         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1087         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1088         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1089         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1090         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1091         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1092
1093         # Apple HTTP Live Streaming
1094         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1095         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1096         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1097         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1098         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1099         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1100         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1101         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1102
1103         # DASH mp4 video
1104         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1105         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1106         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1107         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1108         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1109         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1110         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1111         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1112         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1113         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1114         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1115         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1116
1117         # Dash mp4 audio
1118         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1119         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1120         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1121         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1122         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1123         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1124         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1125
1126         # Dash webm
1127         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1128         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1129         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1130         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1131         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1132         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1133         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1134         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1135         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1136         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1137         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1138         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1139         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1140         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1141         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1142         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1143         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1144         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1145         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1146         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1147         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1148         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1149
1150         # Dash webm audio
1151         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1152         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1153
1154         # Dash webm audio with opus inside
1155         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1156         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1157         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1158
1159         # RTMP (unnamed)
1160         '_rtmp': {'protocol': 'rtmp'},
1161
1162         # av01 video only formats sometimes served with "unknown" codecs
1163         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1164         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1165         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1166         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1167         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1168         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1169         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1170         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1171     }
1172     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1173
1174     _GEO_BYPASS = False
1175
1176     IE_NAME = 'youtube'
1177     _TESTS = [
1178         {
1179             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1180             'info_dict': {
1181                 'id': 'BaW_jenozKc',
1182                 'ext': 'mp4',
1183                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1184                 'uploader': 'Philipp Hagemeister',
1185                 'uploader_id': 'phihag',
1186                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1187                 'channel': 'Philipp Hagemeister',
1188                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1189                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1190                 'upload_date': '20121002',
1191                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1192                 'categories': ['Science & Technology'],
1193                 'tags': ['youtube-dl'],
1194                 'duration': 10,
1195                 'view_count': int,
1196                 'like_count': int,
1197                 'availability': 'public',
1198                 'playable_in_embed': True,
1199                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1200                 'live_status': 'not_live',
1201                 'age_limit': 0,
1202                 'start_time': 1,
1203                 'end_time': 9,
1204                 'comment_count': int,
1205                 'channel_follower_count': int
1206             }
1207         },
1208         {
1209             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1210             'note': 'Embed-only video (#1746)',
1211             'info_dict': {
1212                 'id': 'yZIXLfi8CZQ',
1213                 'ext': 'mp4',
1214                 'upload_date': '20120608',
1215                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1216                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1217                 'uploader': 'SET India',
1218                 'uploader_id': 'setindia',
1219                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1220                 'age_limit': 18,
1221             },
1222             'skip': 'Private video',
1223         },
1224         {
1225             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1226             'note': 'Use the first video ID in the URL',
1227             'info_dict': {
1228                 'id': 'BaW_jenozKc',
1229                 'ext': 'mp4',
1230                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1231                 'uploader': 'Philipp Hagemeister',
1232                 'uploader_id': 'phihag',
1233                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1234                 'channel': 'Philipp Hagemeister',
1235                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1236                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1237                 'upload_date': '20121002',
1238                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1239                 'categories': ['Science & Technology'],
1240                 'tags': ['youtube-dl'],
1241                 'duration': 10,
1242                 'view_count': int,
1243                 'like_count': int,
1244                 'availability': 'public',
1245                 'playable_in_embed': True,
1246                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1247                 'live_status': 'not_live',
1248                 'age_limit': 0,
1249                 'comment_count': int,
1250                 'channel_follower_count': int
1251             },
1252             'params': {
1253                 'skip_download': True,
1254             },
1255         },
1256         {
1257             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1258             'note': '256k DASH audio (format 141) via DASH manifest',
1259             'info_dict': {
1260                 'id': 'a9LDPn-MO4I',
1261                 'ext': 'm4a',
1262                 'upload_date': '20121002',
1263                 'uploader_id': '8KVIDEO',
1264                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1265                 'description': '',
1266                 'uploader': '8KVIDEO',
1267                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1268             },
1269             'params': {
1270                 'youtube_include_dash_manifest': True,
1271                 'format': '141',
1272             },
1273             'skip': 'format 141 not served anymore',
1274         },
1275         # DASH manifest with encrypted signature
1276         {
1277             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1278             'info_dict': {
1279                 'id': 'IB3lcPjvWLA',
1280                 'ext': 'm4a',
1281                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1282                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1283                 'duration': 244,
1284                 'uploader': 'AfrojackVEVO',
1285                 'uploader_id': 'AfrojackVEVO',
1286                 'upload_date': '20131011',
1287                 'abr': 129.495,
1288                 'like_count': int,
1289                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1290                 'playable_in_embed': True,
1291                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1292                 'view_count': int,
1293                 'track': 'The Spark',
1294                 'live_status': 'not_live',
1295                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1296                 'channel': 'Afrojack',
1297                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1298                 'tags': 'count:19',
1299                 'availability': 'public',
1300                 'categories': ['Music'],
1301                 'age_limit': 0,
1302                 'alt_title': 'The Spark',
1303                 'channel_follower_count': int
1304             },
1305             'params': {
1306                 'youtube_include_dash_manifest': True,
1307                 'format': '141/bestaudio[ext=m4a]',
1308             },
1309         },
1310         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1311         {
1312             'note': 'Embed allowed age-gate video',
1313             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1314             'info_dict': {
1315                 'id': 'HtVdAasjOgU',
1316                 'ext': 'mp4',
1317                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1318                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1319                 'duration': 142,
1320                 'uploader': 'The Witcher',
1321                 'uploader_id': 'WitcherGame',
1322                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1323                 'upload_date': '20140605',
1324                 'age_limit': 18,
1325                 'categories': ['Gaming'],
1326                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1327                 'availability': 'needs_auth',
1328                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1329                 'like_count': int,
1330                 'channel': 'The Witcher',
1331                 'live_status': 'not_live',
1332                 'tags': 'count:17',
1333                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1334                 'playable_in_embed': True,
1335                 'view_count': int,
1336                 'channel_follower_count': int
1337             },
1338         },
1339         {
1340             'note': 'Age-gate video with embed allowed in public site',
1341             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1342             'info_dict': {
1343                 'id': 'HsUATh_Nc2U',
1344                 'ext': 'mp4',
1345                 'title': 'Godzilla 2 (Official Video)',
1346                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1347                 'upload_date': '20200408',
1348                 'uploader_id': 'FlyingKitty900',
1349                 'uploader': 'FlyingKitty',
1350                 'age_limit': 18,
1351                 'availability': 'needs_auth',
1352                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1353                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1354                 'channel': 'FlyingKitty',
1355                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1356                 'view_count': int,
1357                 'categories': ['Entertainment'],
1358                 'live_status': 'not_live',
1359                 'tags': ['Flyingkitty', 'godzilla 2'],
1360                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1361                 'like_count': int,
1362                 'duration': 177,
1363                 'playable_in_embed': True,
1364                 'channel_follower_count': int
1365             },
1366         },
1367         {
1368             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1369             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1370             'info_dict': {
1371                 'id': 'Tq92D6wQ1mg',
1372                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1373                 'ext': 'mp4',
1374                 'upload_date': '20191228',
1375                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1376                 'uploader': 'Projekt Melody',
1377                 'description': 'md5:17eccca93a786d51bc67646756894066',
1378                 'age_limit': 18,
1379                 'like_count': int,
1380                 'availability': 'needs_auth',
1381                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1382                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1383                 'view_count': int,
1384                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1385                 'channel': 'Projekt Melody',
1386                 'live_status': 'not_live',
1387                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1388                 'playable_in_embed': True,
1389                 'categories': ['Entertainment'],
1390                 'duration': 106,
1391                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1392                 'comment_count': int,
1393                 'channel_follower_count': int
1394             },
1395         },
1396         {
1397             'note': 'Non-Agegated non-embeddable video',
1398             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1399             'info_dict': {
1400                 'id': 'MeJVWBSsPAY',
1401                 'ext': 'mp4',
1402                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1403                 'uploader': 'Herr Lurik',
1404                 'uploader_id': 'st3in234',
1405                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1406                 'upload_date': '20130730',
1407                 'track': 'Such mich find mich',
1408                 'age_limit': 0,
1409                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1410                 'like_count': int,
1411                 'playable_in_embed': False,
1412                 'creator': 'OOMPH!',
1413                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1414                 'view_count': int,
1415                 'alt_title': 'Such mich find mich',
1416                 'duration': 210,
1417                 'channel': 'Herr Lurik',
1418                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1419                 'categories': ['Music'],
1420                 'availability': 'public',
1421                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1422                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1423                 'live_status': 'not_live',
1424                 'artist': 'OOMPH!',
1425                 'channel_follower_count': int
1426             },
1427         },
1428         {
1429             'note': 'Non-bypassable age-gated video',
1430             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1431             'only_matching': True,
1432         },
1433         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1434         # YouTube Red ad is not captured for creator
1435         {
1436             'url': '__2ABJjxzNo',
1437             'info_dict': {
1438                 'id': '__2ABJjxzNo',
1439                 'ext': 'mp4',
1440                 'duration': 266,
1441                 'upload_date': '20100430',
1442                 'uploader_id': 'deadmau5',
1443                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1444                 'creator': 'deadmau5',
1445                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1446                 'uploader': 'deadmau5',
1447                 'title': 'Deadmau5 - Some Chords (HD)',
1448                 'alt_title': 'Some Chords',
1449                 'availability': 'public',
1450                 'tags': 'count:14',
1451                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1452                 'view_count': int,
1453                 'live_status': 'not_live',
1454                 'channel': 'deadmau5',
1455                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1456                 'like_count': int,
1457                 'track': 'Some Chords',
1458                 'artist': 'deadmau5',
1459                 'playable_in_embed': True,
1460                 'age_limit': 0,
1461                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1462                 'categories': ['Music'],
1463                 'album': 'Some Chords',
1464                 'channel_follower_count': int
1465             },
1466             'expected_warnings': [
1467                 'DASH manifest missing',
1468             ]
1469         },
1470         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1471         {
1472             'url': 'lqQg6PlCWgI',
1473             'info_dict': {
1474                 'id': 'lqQg6PlCWgI',
1475                 'ext': 'mp4',
1476                 'duration': 6085,
1477                 'upload_date': '20150827',
1478                 'uploader_id': 'olympic',
1479                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1480                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1481                 'uploader': 'Olympics',
1482                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1483                 'like_count': int,
1484                 'release_timestamp': 1343767800,
1485                 'playable_in_embed': True,
1486                 'categories': ['Sports'],
1487                 'release_date': '20120731',
1488                 'channel': 'Olympics',
1489                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1490                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1491                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1492                 'age_limit': 0,
1493                 'availability': 'public',
1494                 'live_status': 'was_live',
1495                 'view_count': int,
1496                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1497                 'channel_follower_count': int
1498             },
1499             'params': {
1500                 'skip_download': 'requires avconv',
1501             }
1502         },
1503         # Non-square pixels
1504         {
1505             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1506             'info_dict': {
1507                 'id': '_b-2C3KPAM0',
1508                 'ext': 'mp4',
1509                 'stretched_ratio': 16 / 9.,
1510                 'duration': 85,
1511                 'upload_date': '20110310',
1512                 'uploader_id': 'AllenMeow',
1513                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1514                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1515                 'uploader': '孫ᄋᄅ',
1516                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1517                 'playable_in_embed': True,
1518                 'channel': '孫ᄋᄅ',
1519                 'age_limit': 0,
1520                 'tags': 'count:11',
1521                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1522                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1523                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1524                 'view_count': int,
1525                 'categories': ['People & Blogs'],
1526                 'like_count': int,
1527                 'live_status': 'not_live',
1528                 'availability': 'unlisted',
1529                 'comment_count': int,
1530                 'channel_follower_count': int
1531             },
1532         },
1533         # url_encoded_fmt_stream_map is empty string
1534         {
1535             'url': 'qEJwOuvDf7I',
1536             'info_dict': {
1537                 'id': 'qEJwOuvDf7I',
1538                 'ext': 'webm',
1539                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1540                 'description': '',
1541                 'upload_date': '20150404',
1542                 'uploader_id': 'spbelect',
1543                 'uploader': 'Наблюдатели Петербурга',
1544             },
1545             'params': {
1546                 'skip_download': 'requires avconv',
1547             },
1548             'skip': 'This live event has ended.',
1549         },
1550         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1551         {
1552             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1553             'info_dict': {
1554                 'id': 'FIl7x6_3R5Y',
1555                 'ext': 'webm',
1556                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1557                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1558                 'duration': 220,
1559                 'upload_date': '20150625',
1560                 'uploader_id': 'dorappi2000',
1561                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1562                 'uploader': 'dorappi2000',
1563                 'formats': 'mincount:31',
1564             },
1565             'skip': 'not actual anymore',
1566         },
1567         # DASH manifest with segment_list
1568         {
1569             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1570             'md5': '8ce563a1d667b599d21064e982ab9e31',
1571             'info_dict': {
1572                 'id': 'CsmdDsKjzN8',
1573                 'ext': 'mp4',
1574                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1575                 'uploader': 'Airtek',
1576                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1577                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1578                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1579             },
1580             'params': {
1581                 'youtube_include_dash_manifest': True,
1582                 'format': '135',  # bestvideo
1583             },
1584             'skip': 'This live event has ended.',
1585         },
1586         {
1587             # Multifeed videos (multiple cameras), URL can be of any Camera
1588             'url': 'https://www.youtube.com/watch?v=zaPI8MvL8pg',
1589             'info_dict': {
1590                 'id': 'zaPI8MvL8pg',
1591                 'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04',
1592                 'description': 'md5:563ccbc698b39298481ca3c571169519',
1593             },
1594             'playlist': [{
1595                 'info_dict': {
1596                     'id': 'j5yGuxZ8lLU',
1597                     'ext': 'mp4',
1598                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Chris)',
1599                     'uploader': 'WiiLikeToPlay',
1600                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1601                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1602                     'duration': 10120,
1603                     'channel_follower_count': int,
1604                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1605                     'availability': 'public',
1606                     'playable_in_embed': True,
1607                     'upload_date': '20131105',
1608                     'uploader_id': 'WiiRikeToPray',
1609                     'categories': ['Gaming'],
1610                     'live_status': 'was_live',
1611                     'tags': 'count:24',
1612                     'release_timestamp': 1383701910,
1613                     'thumbnail': 'https://i.ytimg.com/vi/j5yGuxZ8lLU/maxresdefault.jpg',
1614                     'comment_count': int,
1615                     'age_limit': 0,
1616                     'like_count': int,
1617                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1618                     'channel': 'WiiLikeToPlay',
1619                     'view_count': int,
1620                     'release_date': '20131106',
1621                 },
1622             }, {
1623                 'info_dict': {
1624                     'id': 'zaPI8MvL8pg',
1625                     'ext': 'mp4',
1626                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Tyson)',
1627                     'uploader_id': 'WiiRikeToPray',
1628                     'availability': 'public',
1629                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1630                     'channel': 'WiiLikeToPlay',
1631                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1632                     'channel_follower_count': int,
1633                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1634                     'duration': 10108,
1635                     'age_limit': 0,
1636                     'like_count': int,
1637                     'tags': 'count:24',
1638                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1639                     'uploader': 'WiiLikeToPlay',
1640                     'release_timestamp': 1383701915,
1641                     'comment_count': int,
1642                     'upload_date': '20131105',
1643                     'thumbnail': 'https://i.ytimg.com/vi/zaPI8MvL8pg/maxresdefault.jpg',
1644                     'release_date': '20131106',
1645                     'playable_in_embed': True,
1646                     'live_status': 'was_live',
1647                     'categories': ['Gaming'],
1648                     'view_count': int,
1649                 },
1650             }, {
1651                 'info_dict': {
1652                     'id': 'R7r3vfO7Hao',
1653                     'ext': 'mp4',
1654                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Spencer)',
1655                     'thumbnail': 'https://i.ytimg.com/vi/R7r3vfO7Hao/maxresdefault.jpg',
1656                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1657                     'like_count': int,
1658                     'availability': 'public',
1659                     'playable_in_embed': True,
1660                     'upload_date': '20131105',
1661                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1662                     'uploader_id': 'WiiRikeToPray',
1663                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1664                     'channel_follower_count': int,
1665                     'tags': 'count:24',
1666                     'release_date': '20131106',
1667                     'uploader': 'WiiLikeToPlay',
1668                     'comment_count': int,
1669                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1670                     'channel': 'WiiLikeToPlay',
1671                     'categories': ['Gaming'],
1672                     'release_timestamp': 1383701914,
1673                     'live_status': 'was_live',
1674                     'age_limit': 0,
1675                     'duration': 10128,
1676                     'view_count': int,
1677                 },
1678             }],
1679             'params': {'skip_download': True},
1680         },
1681         {
1682             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1683             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1684             'info_dict': {
1685                 'id': 'gVfLd0zydlo',
1686                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1687             },
1688             'playlist_count': 2,
1689             'skip': 'Not multifeed anymore',
1690         },
1691         {
1692             'url': 'https://vid.plus/FlRa-iH7PGw',
1693             'only_matching': True,
1694         },
1695         {
1696             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1697             'only_matching': True,
1698         },
1699         {
1700             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1701             # Also tests cut-off URL expansion in video description (see
1702             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1703             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1704             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1705             'info_dict': {
1706                 'id': 'lsguqyKfVQg',
1707                 'ext': 'mp4',
1708                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1709                 'alt_title': 'Dark Walk',
1710                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1711                 'duration': 133,
1712                 'upload_date': '20151119',
1713                 'uploader_id': 'IronSoulElf',
1714                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1715                 'uploader': 'IronSoulElf',
1716                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1717                 'track': 'Dark Walk',
1718                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1719                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1720                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1721                 'categories': ['Film & Animation'],
1722                 'view_count': int,
1723                 'live_status': 'not_live',
1724                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1725                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1726                 'tags': 'count:13',
1727                 'availability': 'public',
1728                 'channel': 'IronSoulElf',
1729                 'playable_in_embed': True,
1730                 'like_count': int,
1731                 'age_limit': 0,
1732                 'channel_follower_count': int
1733             },
1734             'params': {
1735                 'skip_download': True,
1736             },
1737         },
1738         {
1739             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1740             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1741             'only_matching': True,
1742         },
1743         {
1744             # Video with yt:stretch=17:0
1745             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1746             'info_dict': {
1747                 'id': 'Q39EVAstoRM',
1748                 'ext': 'mp4',
1749                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1750                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1751                 'upload_date': '20151107',
1752                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1753                 'uploader': 'CH GAMER DROID',
1754             },
1755             'params': {
1756                 'skip_download': True,
1757             },
1758             'skip': 'This video does not exist.',
1759         },
1760         {
1761             # Video with incomplete 'yt:stretch=16:'
1762             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1763             'only_matching': True,
1764         },
1765         {
1766             # Video licensed under Creative Commons
1767             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1768             'info_dict': {
1769                 'id': 'M4gD1WSo5mA',
1770                 'ext': 'mp4',
1771                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1772                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1773                 'duration': 721,
1774                 'upload_date': '20150128',
1775                 'uploader_id': 'BerkmanCenter',
1776                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1777                 'uploader': 'The Berkman Klein Center for Internet & Society',
1778                 'license': 'Creative Commons Attribution license (reuse allowed)',
1779                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1780                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1781                 'like_count': int,
1782                 'age_limit': 0,
1783                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1784                 'channel': 'The Berkman Klein Center for Internet & Society',
1785                 'availability': 'public',
1786                 'view_count': int,
1787                 'categories': ['Education'],
1788                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1789                 'live_status': 'not_live',
1790                 'playable_in_embed': True,
1791                 'comment_count': int,
1792                 'channel_follower_count': int,
1793                 'chapters': list,
1794             },
1795             'params': {
1796                 'skip_download': True,
1797             },
1798         },
1799         {
1800             # Channel-like uploader_url
1801             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1802             'info_dict': {
1803                 'id': 'eQcmzGIKrzg',
1804                 'ext': 'mp4',
1805                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1806                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1807                 'duration': 4060,
1808                 'upload_date': '20151120',
1809                 'uploader': 'Bernie Sanders',
1810                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1811                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1812                 'license': 'Creative Commons Attribution license (reuse allowed)',
1813                 'playable_in_embed': True,
1814                 'tags': 'count:12',
1815                 'like_count': int,
1816                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1817                 'age_limit': 0,
1818                 'availability': 'public',
1819                 'categories': ['News & Politics'],
1820                 'channel': 'Bernie Sanders',
1821                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1822                 'view_count': int,
1823                 'live_status': 'not_live',
1824                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1825                 'comment_count': int,
1826                 'channel_follower_count': int,
1827                 'chapters': list,
1828             },
1829             'params': {
1830                 'skip_download': True,
1831             },
1832         },
1833         {
1834             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1835             'only_matching': True,
1836         },
1837         {
1838             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1839             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1840             'only_matching': True,
1841         },
1842         {
1843             # Rental video preview
1844             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1845             'info_dict': {
1846                 'id': 'uGpuVWrhIzE',
1847                 'ext': 'mp4',
1848                 'title': 'Piku - Trailer',
1849                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1850                 'upload_date': '20150811',
1851                 'uploader': 'FlixMatrix',
1852                 'uploader_id': 'FlixMatrixKaravan',
1853                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1854                 'license': 'Standard YouTube License',
1855             },
1856             'params': {
1857                 'skip_download': True,
1858             },
1859             'skip': 'This video is not available.',
1860         },
1861         {
1862             # YouTube Red video with episode data
1863             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1864             'info_dict': {
1865                 'id': 'iqKdEhx-dD4',
1866                 'ext': 'mp4',
1867                 'title': 'Isolation - Mind Field (Ep 1)',
1868                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1869                 'duration': 2085,
1870                 'upload_date': '20170118',
1871                 'uploader': 'Vsauce',
1872                 'uploader_id': 'Vsauce',
1873                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1874                 'series': 'Mind Field',
1875                 'season_number': 1,
1876                 'episode_number': 1,
1877                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1878                 'tags': 'count:12',
1879                 'view_count': int,
1880                 'availability': 'public',
1881                 'age_limit': 0,
1882                 'channel': 'Vsauce',
1883                 'episode': 'Episode 1',
1884                 'categories': ['Entertainment'],
1885                 'season': 'Season 1',
1886                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1887                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1888                 'like_count': int,
1889                 'playable_in_embed': True,
1890                 'live_status': 'not_live',
1891                 'channel_follower_count': int
1892             },
1893             'params': {
1894                 'skip_download': True,
1895             },
1896             'expected_warnings': [
1897                 'Skipping DASH manifest',
1898             ],
1899         },
1900         {
1901             # The following content has been identified by the YouTube community
1902             # as inappropriate or offensive to some audiences.
1903             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1904             'info_dict': {
1905                 'id': '6SJNVb0GnPI',
1906                 'ext': 'mp4',
1907                 'title': 'Race Differences in Intelligence',
1908                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1909                 'duration': 965,
1910                 'upload_date': '20140124',
1911                 'uploader': 'New Century Foundation',
1912                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1913                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1914             },
1915             'params': {
1916                 'skip_download': True,
1917             },
1918             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1919         },
1920         {
1921             # itag 212
1922             'url': '1t24XAntNCY',
1923             'only_matching': True,
1924         },
1925         {
1926             # geo restricted to JP
1927             'url': 'sJL6WA-aGkQ',
1928             'only_matching': True,
1929         },
1930         {
1931             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1932             'only_matching': True,
1933         },
1934         {
1935             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1936             'only_matching': True,
1937         },
1938         {
1939             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1940             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1941             'only_matching': True,
1942         },
1943         {
1944             # DRM protected
1945             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1946             'only_matching': True,
1947         },
1948         {
1949             # Video with unsupported adaptive stream type formats
1950             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1951             'info_dict': {
1952                 'id': 'Z4Vy8R84T1U',
1953                 'ext': 'mp4',
1954                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1955                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1956                 'duration': 433,
1957                 'upload_date': '20130923',
1958                 'uploader': 'Amelia Putri Harwita',
1959                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1960                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1961                 'formats': 'maxcount:10',
1962             },
1963             'params': {
1964                 'skip_download': True,
1965                 'youtube_include_dash_manifest': False,
1966             },
1967             'skip': 'not actual anymore',
1968         },
1969         {
1970             # Youtube Music Auto-generated description
1971             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1972             'info_dict': {
1973                 'id': 'MgNrAu2pzNs',
1974                 'ext': 'mp4',
1975                 'title': 'Voyeur Girl',
1976                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1977                 'upload_date': '20190312',
1978                 'uploader': 'Stephen - Topic',
1979                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1980                 'artist': 'Stephen',
1981                 'track': 'Voyeur Girl',
1982                 'album': 'it\'s too much love to know my dear',
1983                 'release_date': '20190313',
1984                 'release_year': 2019,
1985                 'alt_title': 'Voyeur Girl',
1986                 'view_count': int,
1987                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1988                 'playable_in_embed': True,
1989                 'like_count': int,
1990                 'categories': ['Music'],
1991                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1992                 'channel': 'Stephen',
1993                 'availability': 'public',
1994                 'creator': 'Stephen',
1995                 'duration': 169,
1996                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1997                 'age_limit': 0,
1998                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1999                 'tags': 'count:11',
2000                 'live_status': 'not_live',
2001                 'channel_follower_count': int
2002             },
2003             'params': {
2004                 'skip_download': True,
2005             },
2006         },
2007         {
2008             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
2009             'only_matching': True,
2010         },
2011         {
2012             # invalid -> valid video id redirection
2013             'url': 'DJztXj2GPfl',
2014             'info_dict': {
2015                 'id': 'DJztXj2GPfk',
2016                 'ext': 'mp4',
2017                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
2018                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
2019                 'upload_date': '20090125',
2020                 'uploader': 'Prochorowka',
2021                 'uploader_id': 'Prochorowka',
2022                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
2023                 'artist': 'Panjabi MC',
2024                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
2025                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
2026             },
2027             'params': {
2028                 'skip_download': True,
2029             },
2030             'skip': 'Video unavailable',
2031         },
2032         {
2033             # empty description results in an empty string
2034             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
2035             'info_dict': {
2036                 'id': 'x41yOUIvK2k',
2037                 'ext': 'mp4',
2038                 'title': 'IMG 3456',
2039                 'description': '',
2040                 'upload_date': '20170613',
2041                 'uploader_id': 'ElevageOrVert',
2042                 'uploader': 'ElevageOrVert',
2043                 'view_count': int,
2044                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
2045                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
2046                 'like_count': int,
2047                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
2048                 'tags': [],
2049                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
2050                 'availability': 'public',
2051                 'age_limit': 0,
2052                 'categories': ['Pets & Animals'],
2053                 'duration': 7,
2054                 'playable_in_embed': True,
2055                 'live_status': 'not_live',
2056                 'channel': 'ElevageOrVert',
2057                 'channel_follower_count': int
2058             },
2059             'params': {
2060                 'skip_download': True,
2061             },
2062         },
2063         {
2064             # with '};' inside yt initial data (see [1])
2065             # see [2] for an example with '};' inside ytInitialPlayerResponse
2066             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2067             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2068             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2069             'info_dict': {
2070                 'id': 'CHqg6qOn4no',
2071                 'ext': 'mp4',
2072                 'title': 'Part 77   Sort a list of simple types in c#',
2073                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2074                 'upload_date': '20130831',
2075                 'uploader_id': 'kudvenkat',
2076                 'uploader': 'kudvenkat',
2077                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2078                 'like_count': int,
2079                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2080                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2081                 'live_status': 'not_live',
2082                 'categories': ['Education'],
2083                 'availability': 'public',
2084                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2085                 'tags': 'count:12',
2086                 'playable_in_embed': True,
2087                 'age_limit': 0,
2088                 'view_count': int,
2089                 'duration': 522,
2090                 'channel': 'kudvenkat',
2091                 'comment_count': int,
2092                 'channel_follower_count': int,
2093                 'chapters': list,
2094             },
2095             'params': {
2096                 'skip_download': True,
2097             },
2098         },
2099         {
2100             # another example of '};' in ytInitialData
2101             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2102             'only_matching': True,
2103         },
2104         {
2105             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2106             'only_matching': True,
2107         },
2108         {
2109             # https://github.com/ytdl-org/youtube-dl/pull/28094
2110             'url': 'OtqTfy26tG0',
2111             'info_dict': {
2112                 'id': 'OtqTfy26tG0',
2113                 'ext': 'mp4',
2114                 'title': 'Burn Out',
2115                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2116                 'upload_date': '20141120',
2117                 'uploader': 'The Cinematic Orchestra - Topic',
2118                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2119                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2120                 'artist': 'The Cinematic Orchestra',
2121                 'track': 'Burn Out',
2122                 'album': 'Every Day',
2123                 'like_count': int,
2124                 'live_status': 'not_live',
2125                 'alt_title': 'Burn Out',
2126                 'duration': 614,
2127                 'age_limit': 0,
2128                 'view_count': int,
2129                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2130                 'creator': 'The Cinematic Orchestra',
2131                 'channel': 'The Cinematic Orchestra',
2132                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2133                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2134                 'availability': 'public',
2135                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2136                 'categories': ['Music'],
2137                 'playable_in_embed': True,
2138                 'channel_follower_count': int
2139             },
2140             'params': {
2141                 'skip_download': True,
2142             },
2143         },
2144         {
2145             # controversial video, only works with bpctr when authenticated with cookies
2146             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2147             'only_matching': True,
2148         },
2149         {
2150             # controversial video, requires bpctr/contentCheckOk
2151             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2152             'info_dict': {
2153                 'id': 'SZJvDhaSDnc',
2154                 'ext': 'mp4',
2155                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2156                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2157                 'uploader': 'CBS Mornings',
2158                 'uploader_id': 'CBSThisMorning',
2159                 'upload_date': '20140716',
2160                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2161                 'duration': 170,
2162                 'categories': ['News & Politics'],
2163                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2164                 'view_count': int,
2165                 'channel': 'CBS Mornings',
2166                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2167                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2168                 'age_limit': 18,
2169                 'availability': 'needs_auth',
2170                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2171                 'like_count': int,
2172                 'live_status': 'not_live',
2173                 'playable_in_embed': True,
2174                 'channel_follower_count': int
2175             }
2176         },
2177         {
2178             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2179             'url': 'cBvYw8_A0vQ',
2180             'info_dict': {
2181                 'id': 'cBvYw8_A0vQ',
2182                 'ext': 'mp4',
2183                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2184                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2185                 'upload_date': '20201120',
2186                 'uploader': 'Walk around Japan',
2187                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2188                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2189                 'duration': 1456,
2190                 'categories': ['Travel & Events'],
2191                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2192                 'view_count': int,
2193                 'channel': 'Walk around Japan',
2194                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2195                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2196                 'age_limit': 0,
2197                 'availability': 'public',
2198                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2199                 'live_status': 'not_live',
2200                 'playable_in_embed': True,
2201                 'channel_follower_count': int
2202             },
2203             'params': {
2204                 'skip_download': True,
2205             },
2206         }, {
2207             # Has multiple audio streams
2208             'url': 'WaOKSUlf4TM',
2209             'only_matching': True
2210         }, {
2211             # Requires Premium: has format 141 when requested using YTM url
2212             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2213             'only_matching': True
2214         }, {
2215             # multiple subtitles with same lang_code
2216             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2217             'only_matching': True,
2218         }, {
2219             # Force use android client fallback
2220             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2221             'info_dict': {
2222                 'id': 'YOelRv7fMxY',
2223                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2224                 'ext': '3gp',
2225                 'upload_date': '20210624',
2226                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2227                 'uploader': 'colinfurze',
2228                 'uploader_id': 'colinfurze',
2229                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2230                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2231                 'duration': 596,
2232                 'categories': ['Entertainment'],
2233                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2234                 'view_count': int,
2235                 'channel': 'colinfurze',
2236                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2237                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2238                 'age_limit': 0,
2239                 'availability': 'public',
2240                 'like_count': int,
2241                 'live_status': 'not_live',
2242                 'playable_in_embed': True,
2243                 'channel_follower_count': int,
2244                 'chapters': list,
2245             },
2246             'params': {
2247                 'format': '17',  # 3gp format available on android
2248                 'extractor_args': {'youtube': {'player_client': ['android']}},
2249             },
2250         },
2251         {
2252             # Skip download of additional client configs (remix client config in this case)
2253             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2254             'only_matching': True,
2255             'params': {
2256                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2257             },
2258         }, {
2259             # shorts
2260             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2261             'only_matching': True,
2262         }, {
2263             'note': 'Storyboards',
2264             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2265             'info_dict': {
2266                 'id': '5KLPxDtMqe8',
2267                 'ext': 'mhtml',
2268                 'format_id': 'sb0',
2269                 'title': 'Your Brain is Plastic',
2270                 'uploader_id': 'scishow',
2271                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2272                 'upload_date': '20140324',
2273                 'uploader': 'SciShow',
2274                 'like_count': int,
2275                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2276                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2277                 'view_count': int,
2278                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2279                 'playable_in_embed': True,
2280                 'tags': 'count:12',
2281                 'uploader_url': 'http://www.youtube.com/user/scishow',
2282                 'availability': 'public',
2283                 'channel': 'SciShow',
2284                 'live_status': 'not_live',
2285                 'duration': 248,
2286                 'categories': ['Education'],
2287                 'age_limit': 0,
2288                 'channel_follower_count': int,
2289                 'chapters': list,
2290             }, 'params': {'format': 'mhtml', 'skip_download': True}
2291         }, {
2292             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2293             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2294             'info_dict': {
2295                 'id': '2NUZ8W2llS4',
2296                 'ext': 'mp4',
2297                 'title': 'The NP that test your phone performance 🙂',
2298                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2299                 'uploader': 'Leon Nguyen',
2300                 'uploader_id': 'VNSXIII',
2301                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2302                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2303                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2304                 'duration': 21,
2305                 'view_count': int,
2306                 'age_limit': 0,
2307                 'categories': ['Gaming'],
2308                 'tags': 'count:23',
2309                 'playable_in_embed': True,
2310                 'live_status': 'not_live',
2311                 'upload_date': '20220103',
2312                 'like_count': int,
2313                 'availability': 'public',
2314                 'channel': 'Leon Nguyen',
2315                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2316                 'comment_count': int,
2317                 'channel_follower_count': int
2318             }
2319         }, {
2320             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2321             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2322             'info_dict': {
2323                 'id': '2NUZ8W2llS4',
2324                 'ext': 'mp4',
2325                 'title': 'The NP that test your phone performance 🙂',
2326                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2327                 'uploader': 'Leon Nguyen',
2328                 'uploader_id': 'VNSXIII',
2329                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2330                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2331                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2332                 'duration': 21,
2333                 'view_count': int,
2334                 'age_limit': 0,
2335                 'categories': ['Gaming'],
2336                 'tags': 'count:23',
2337                 'playable_in_embed': True,
2338                 'live_status': 'not_live',
2339                 'upload_date': '20220102',
2340                 'like_count': int,
2341                 'availability': 'public',
2342                 'channel': 'Leon Nguyen',
2343                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2344                 'comment_count': int,
2345                 'channel_follower_count': int
2346             },
2347             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2348         }, {
2349             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2350             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2351             'info_dict': {
2352                 'id': 'mzZzzBU6lrM',
2353                 'ext': 'mp4',
2354                 'title': 'I Met GeorgeNotFound In Real Life...',
2355                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2356                 'uploader': 'Quackity',
2357                 'uploader_id': 'QuackityHQ',
2358                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2359                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2360                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2361                 'duration': 955,
2362                 'view_count': int,
2363                 'age_limit': 0,
2364                 'categories': ['Entertainment'],
2365                 'tags': 'count:26',
2366                 'playable_in_embed': True,
2367                 'live_status': 'not_live',
2368                 'release_timestamp': 1641172509,
2369                 'release_date': '20220103',
2370                 'upload_date': '20220103',
2371                 'like_count': int,
2372                 'availability': 'public',
2373                 'channel': 'Quackity',
2374                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2375                 'channel_follower_count': int
2376             }
2377         },
2378         {   # continuous livestream. Microformat upload date should be preferred.
2379             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2380             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2381             'info_dict': {
2382                 'id': 'kgx4WGK0oNU',
2383                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2384                 'ext': 'mp4',
2385                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2386                 'availability': 'public',
2387                 'age_limit': 0,
2388                 'release_timestamp': 1637975704,
2389                 'upload_date': '20210619',
2390                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2391                 'live_status': 'is_live',
2392                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2393                 'uploader': '阿鲍Abao',
2394                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2395                 'channel': 'Abao in Tokyo',
2396                 'channel_follower_count': int,
2397                 'release_date': '20211127',
2398                 'tags': 'count:39',
2399                 'categories': ['People & Blogs'],
2400                 'like_count': int,
2401                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2402                 'view_count': int,
2403                 'playable_in_embed': True,
2404                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2405                 'concurrent_view_count': int,
2406             },
2407             'params': {'skip_download': True}
2408         }, {
2409             # Story. Requires specific player params to work.
2410             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2411             'info_dict': {
2412                 'id': 'vv8qTUWmulI',
2413                 'ext': 'mp4',
2414                 'availability': 'unlisted',
2415                 'view_count': int,
2416                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2417                 'upload_date': '20220526',
2418                 'categories': ['Education'],
2419                 'title': 'Story',
2420                 'channel': 'IT\'S HISTORY',
2421                 'description': '',
2422                 'uploader_id': 'BlastfromthePast',
2423                 'duration': 12,
2424                 'uploader': 'IT\'S HISTORY',
2425                 'playable_in_embed': True,
2426                 'age_limit': 0,
2427                 'live_status': 'not_live',
2428                 'tags': [],
2429                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2430                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2431                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2432             },
2433             'skip': 'stories get removed after some period of time',
2434         }, {
2435             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2436             'info_dict': {
2437                 'id': 'tjjjtzRLHvA',
2438                 'ext': 'mp4',
2439                 'title': 'ハッシュタグ無し };if window.ytcsi',
2440                 'upload_date': '20220323',
2441                 'like_count': int,
2442                 'availability': 'unlisted',
2443                 'channel': 'nao20010128nao',
2444                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2445                 'age_limit': 0,
2446                 'uploader': 'nao20010128nao',
2447                 'uploader_id': 'nao20010128nao',
2448                 'categories': ['Music'],
2449                 'view_count': int,
2450                 'description': '',
2451                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2452                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2453                 'live_status': 'not_live',
2454                 'playable_in_embed': True,
2455                 'channel_follower_count': int,
2456                 'duration': 6,
2457                 'tags': [],
2458                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2459             }
2460         }, {
2461             # Prefer primary title+description language metadata by default
2462             # Do not prefer translated description if primary is empty
2463             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2464             'info_dict': {
2465                 'id': 'el3E4MbxRqQ',
2466                 'ext': 'mp4',
2467                 'title': 'dlp test video 2 - primary sv no desc',
2468                 'description': '',
2469                 'channel': 'cole-dlp-test-acc',
2470                 'tags': [],
2471                 'view_count': int,
2472                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2473                 'like_count': int,
2474                 'playable_in_embed': True,
2475                 'availability': 'unlisted',
2476                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2477                 'age_limit': 0,
2478                 'duration': 5,
2479                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2480                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2481                 'live_status': 'not_live',
2482                 'upload_date': '20220908',
2483                 'categories': ['People & Blogs'],
2484                 'uploader': 'cole-dlp-test-acc',
2485                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2486             },
2487             'params': {'skip_download': True}
2488         }, {
2489             # Extractor argument: prefer translated title+description
2490             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2491             'info_dict': {
2492                 'id': 'gHKT4uU8Zng',
2493                 'ext': 'mp4',
2494                 'channel': 'cole-dlp-test-acc',
2495                 'tags': [],
2496                 'duration': 5,
2497                 'live_status': 'not_live',
2498                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2499                 'upload_date': '20220728',
2500                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2501                 'view_count': int,
2502                 'categories': ['People & Blogs'],
2503                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2504                 'title': 'dlp test video title translated (fr)',
2505                 'availability': 'public',
2506                 'uploader': 'cole-dlp-test-acc',
2507                 'age_limit': 0,
2508                 'description': 'dlp test video description translated (fr)',
2509                 'playable_in_embed': True,
2510                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2511                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2512             },
2513             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2514             'expected_warnings': [r'Preferring "fr" translated fields'],
2515         }, {
2516             'note': '6 channel audio',
2517             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2518             'only_matching': True,
2519         }, {
2520             'note': 'Multiple HLS formats with same itag',
2521             'url': 'https://www.youtube.com/watch?v=kX3nB4PpJko',
2522             'info_dict': {
2523                 'id': 'kX3nB4PpJko',
2524                 'ext': 'mp4',
2525                 'categories': ['Entertainment'],
2526                 'description': 'md5:e8031ff6e426cdb6a77670c9b81f6fa6',
2527                 'uploader_url': 'http://www.youtube.com/user/MrBeast6000',
2528                 'live_status': 'not_live',
2529                 'duration': 937,
2530                 'channel_follower_count': int,
2531                 'thumbnail': 'https://i.ytimg.com/vi_webp/kX3nB4PpJko/maxresdefault.webp',
2532                 'title': 'Last To Take Hand Off Jet, Keeps It!',
2533                 'channel': 'MrBeast',
2534                 'playable_in_embed': True,
2535                 'view_count': int,
2536                 'upload_date': '20221112',
2537                 'uploader': 'MrBeast',
2538                 'uploader_id': 'MrBeast6000',
2539                 'channel_url': 'https://www.youtube.com/channel/UCX6OQ3DkcsbYNE6H8uQQuVA',
2540                 'age_limit': 0,
2541                 'availability': 'public',
2542                 'channel_id': 'UCX6OQ3DkcsbYNE6H8uQQuVA',
2543                 'like_count': int,
2544                 'tags': [],
2545             },
2546             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
2547         }, {
2548             'note': 'Audio formats with Dynamic Range Compression',
2549             'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
2550             'info_dict': {
2551                 'id': 'Tq92D6wQ1mg',
2552                 'ext': 'weba',
2553                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
2554                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2555                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2556                 'channel_follower_count': int,
2557                 'description': 'md5:17eccca93a786d51bc67646756894066',
2558                 'upload_date': '20191228',
2559                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2560                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
2561                 'playable_in_embed': True,
2562                 'like_count': int,
2563                 'categories': ['Entertainment'],
2564                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
2565                 'age_limit': 18,
2566                 'channel': 'Projekt Melody',
2567                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2568                 'view_count': int,
2569                 'availability': 'needs_auth',
2570                 'comment_count': int,
2571                 'live_status': 'not_live',
2572                 'uploader': 'Projekt Melody',
2573                 'duration': 106,
2574             },
2575             'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
2576         }
2577     ]
2578
2579     _WEBPAGE_TESTS = [
2580         # YouTube <object> embed
2581         {
2582             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2583             'md5': '873c81d308b979f0e23ee7e620b312a3',
2584             'info_dict': {
2585                 'id': 'msN87y-iEx0',
2586                 'ext': 'mp4',
2587                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2588                 'upload_date': '20080526',
2589                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2590                 'uploader': 'Christopher Sykes',
2591                 'uploader_id': 'ChristopherJSykes',
2592                 'age_limit': 0,
2593                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2594                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2595                 'playable_in_embed': True,
2596                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2597                 'like_count': int,
2598                 'comment_count': int,
2599                 'channel': 'Christopher Sykes',
2600                 'live_status': 'not_live',
2601                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2602                 'availability': 'public',
2603                 'duration': 195,
2604                 'view_count': int,
2605                 'categories': ['Science & Technology'],
2606                 'channel_follower_count': int,
2607                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2608             },
2609             'params': {
2610                 'skip_download': True,
2611             }
2612         },
2613     ]
2614
2615     @classmethod
2616     def suitable(cls, url):
2617         from ..utils import parse_qs
2618
2619         qs = parse_qs(url)
2620         if qs.get('list', [None])[0]:
2621             return False
2622         return super().suitable(url)
2623
2624     def __init__(self, *args, **kwargs):
2625         super().__init__(*args, **kwargs)
2626         self._code_cache = {}
2627         self._player_cache = {}
2628
2629     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2630         lock = threading.Lock()
2631         start_time = time.time()
2632         formats = [f for f in formats if f.get('is_from_start')]
2633
2634         def refetch_manifest(format_id, delay):
2635             nonlocal formats, start_time, is_live
2636             if time.time() <= start_time + delay:
2637                 return
2638
2639             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2640             video_details = traverse_obj(
2641                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2642             microformats = traverse_obj(
2643                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2644                 expected_type=dict, default=[])
2645             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2646             is_live = live_status == 'is_live'
2647             start_time = time.time()
2648
2649         def mpd_feed(format_id, delay):
2650             """
2651             @returns (manifest_url, manifest_stream_number, is_live) or None
2652             """
2653             for retry in self.RetryManager(fatal=False):
2654                 with lock:
2655                     refetch_manifest(format_id, delay)
2656
2657                 f = next((f for f in formats if f['format_id'] == format_id), None)
2658                 if not f:
2659                     if not is_live:
2660                         retry.error = f'{video_id}: Video is no longer live'
2661                     else:
2662                         retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
2663                     continue
2664                 return f['manifest_url'], f['manifest_stream_number'], is_live
2665             return None
2666
2667         for f in formats:
2668             f['is_live'] = is_live
2669             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2670                                     live_start_time, mpd_feed, not is_live and f.copy())
2671             if is_live:
2672                 f['fragments'] = gen
2673                 f['protocol'] = 'http_dash_segments_generator'
2674             else:
2675                 f['fragments'] = LazyList(gen({}))
2676                 del f['is_from_start']
2677
2678     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2679         FETCH_SPAN, MAX_DURATION = 5, 432000
2680
2681         mpd_url, stream_number, is_live = None, None, True
2682
2683         begin_index = 0
2684         download_start_time = ctx.get('start') or time.time()
2685
2686         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2687         if lack_early_segments:
2688             self.report_warning(bug_reports_message(
2689                 'Starting download from the last 120 hours of the live stream since '
2690                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2691             lack_early_segments = True
2692
2693         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2694         fragments, fragment_base_url = None, None
2695
2696         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2697             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2698             # Obtain from MPD's maximum seq value
2699             old_mpd_url = mpd_url
2700             last_error = ctx.pop('last_error', None)
2701             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2702             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2703                                                or (mpd_url, stream_number, False))
2704             if not refresh_sequence:
2705                 if expire_fast and not is_live:
2706                     return False, last_seq
2707                 elif old_mpd_url == mpd_url:
2708                     return True, last_seq
2709             if manifestless_orig_fmt:
2710                 fmt_info = manifestless_orig_fmt
2711             else:
2712                 try:
2713                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2714                         mpd_url, None, note=False, errnote=False, fatal=False)
2715                 except ExtractorError:
2716                     fmts = None
2717                 if not fmts:
2718                     no_fragment_score += 2
2719                     return False, last_seq
2720                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2721             fragments = fmt_info['fragments']
2722             fragment_base_url = fmt_info['fragment_base_url']
2723             assert fragment_base_url
2724
2725             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2726             return True, _last_seq
2727
2728         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2729         while is_live:
2730             fetch_time = time.time()
2731             if no_fragment_score > 30:
2732                 return
2733             if last_segment_url:
2734                 # Obtain from "X-Head-Seqnum" header value from each segment
2735                 try:
2736                     urlh = self._request_webpage(
2737                         last_segment_url, None, note=False, errnote=False, fatal=False)
2738                 except ExtractorError:
2739                     urlh = None
2740                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2741                 if last_seq is None:
2742                     no_fragment_score += 2
2743                     last_segment_url = None
2744                     continue
2745             else:
2746                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2747                 no_fragment_score += 2
2748                 if not should_continue:
2749                     continue
2750
2751             if known_idx > last_seq:
2752                 last_segment_url = None
2753                 continue
2754
2755             last_seq += 1
2756
2757             if begin_index < 0 and known_idx < 0:
2758                 # skip from the start when it's negative value
2759                 known_idx = last_seq + begin_index
2760             if lack_early_segments:
2761                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2762             try:
2763                 for idx in range(known_idx, last_seq):
2764                     # do not update sequence here or you'll get skipped some part of it
2765                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2766                     if not should_continue:
2767                         known_idx = idx - 1
2768                         raise ExtractorError('breaking out of outer loop')
2769                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2770                     yield {
2771                         'url': last_segment_url,
2772                         'fragment_count': last_seq,
2773                     }
2774                 if known_idx == last_seq:
2775                     no_fragment_score += 5
2776                 else:
2777                     no_fragment_score = 0
2778                 known_idx = last_seq
2779             except ExtractorError:
2780                 continue
2781
2782             if manifestless_orig_fmt:
2783                 # Stop at the first iteration if running for post-live manifestless;
2784                 # fragment count no longer increase since it starts
2785                 break
2786
2787             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2788
2789     def _extract_player_url(self, *ytcfgs, webpage=None):
2790         player_url = traverse_obj(
2791             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2792             get_all=False, expected_type=str)
2793         if not player_url:
2794             return
2795         return urljoin('https://www.youtube.com', player_url)
2796
2797     def _download_player_url(self, video_id, fatal=False):
2798         res = self._download_webpage(
2799             'https://www.youtube.com/iframe_api',
2800             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2801         if res:
2802             player_version = self._search_regex(
2803                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2804             if player_version:
2805                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2806
2807     def _signature_cache_id(self, example_sig):
2808         """ Return a string representation of a signature """
2809         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2810
2811     @classmethod
2812     def _extract_player_info(cls, player_url):
2813         for player_re in cls._PLAYER_INFO_RE:
2814             id_m = re.search(player_re, player_url)
2815             if id_m:
2816                 break
2817         else:
2818             raise ExtractorError('Cannot identify player %r' % player_url)
2819         return id_m.group('id')
2820
2821     def _load_player(self, video_id, player_url, fatal=True):
2822         player_id = self._extract_player_info(player_url)
2823         if player_id not in self._code_cache:
2824             code = self._download_webpage(
2825                 player_url, video_id, fatal=fatal,
2826                 note='Downloading player ' + player_id,
2827                 errnote='Download of %s failed' % player_url)
2828             if code:
2829                 self._code_cache[player_id] = code
2830         return self._code_cache.get(player_id)
2831
2832     def _extract_signature_function(self, video_id, player_url, example_sig):
2833         player_id = self._extract_player_info(player_url)
2834
2835         # Read from filesystem cache
2836         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2837         assert os.path.basename(func_id) == func_id
2838
2839         self.write_debug(f'Extracting signature function {func_id}')
2840         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2841
2842         if not cache_spec:
2843             code = self._load_player(video_id, player_url)
2844         if code:
2845             res = self._parse_sig_js(code)
2846             test_string = ''.join(map(chr, range(len(example_sig))))
2847             cache_spec = [ord(c) for c in res(test_string)]
2848             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2849
2850         return lambda s: ''.join(s[i] for i in cache_spec)
2851
2852     def _print_sig_code(self, func, example_sig):
2853         if not self.get_param('youtube_print_sig_code'):
2854             return
2855
2856         def gen_sig_code(idxs):
2857             def _genslice(start, end, step):
2858                 starts = '' if start == 0 else str(start)
2859                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2860                 steps = '' if step == 1 else (':%d' % step)
2861                 return f's[{starts}{ends}{steps}]'
2862
2863             step = None
2864             # Quelch pyflakes warnings - start will be set when step is set
2865             start = '(Never used)'
2866             for i, prev in zip(idxs[1:], idxs[:-1]):
2867                 if step is not None:
2868                     if i - prev == step:
2869                         continue
2870                     yield _genslice(start, prev, step)
2871                     step = None
2872                     continue
2873                 if i - prev in [-1, 1]:
2874                     step = i - prev
2875                     start = prev
2876                     continue
2877                 else:
2878                     yield 's[%d]' % prev
2879             if step is None:
2880                 yield 's[%d]' % i
2881             else:
2882                 yield _genslice(start, i, step)
2883
2884         test_string = ''.join(map(chr, range(len(example_sig))))
2885         cache_res = func(test_string)
2886         cache_spec = [ord(c) for c in cache_res]
2887         expr_code = ' + '.join(gen_sig_code(cache_spec))
2888         signature_id_tuple = '(%s)' % (
2889             ', '.join(str(len(p)) for p in example_sig.split('.')))
2890         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2891                 '    return %s\n') % (signature_id_tuple, expr_code)
2892         self.to_screen('Extracted signature function:\n' + code)
2893
2894     def _parse_sig_js(self, jscode):
2895         funcname = self._search_regex(
2896             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2897              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2898              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2899              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2900              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2901              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2902              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2903              # Obsolete patterns
2904              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2905              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2906              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2907              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2908              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2909              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2910              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2911              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2912             jscode, 'Initial JS player signature function name', group='sig')
2913
2914         jsi = JSInterpreter(jscode)
2915         initial_function = jsi.extract_function(funcname)
2916         return lambda s: initial_function([s])
2917
2918     def _cached(self, func, *cache_id):
2919         def inner(*args, **kwargs):
2920             if cache_id not in self._player_cache:
2921                 try:
2922                     self._player_cache[cache_id] = func(*args, **kwargs)
2923                 except ExtractorError as e:
2924                     self._player_cache[cache_id] = e
2925                 except Exception as e:
2926                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
2927
2928             ret = self._player_cache[cache_id]
2929             if isinstance(ret, Exception):
2930                 raise ret
2931             return ret
2932         return inner
2933
2934     def _decrypt_signature(self, s, video_id, player_url):
2935         """Turn the encrypted s field into a working signature"""
2936         extract_sig = self._cached(
2937             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
2938         func = extract_sig(video_id, player_url, s)
2939         self._print_sig_code(func, s)
2940         return func(s)
2941
2942     def _decrypt_nsig(self, s, video_id, player_url):
2943         """Turn the encrypted n field into a working signature"""
2944         if player_url is None:
2945             raise ExtractorError('Cannot decrypt nsig without player_url')
2946         player_url = urljoin('https://www.youtube.com', player_url)
2947
2948         try:
2949             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
2950         except ExtractorError as e:
2951             raise ExtractorError('Unable to extract nsig function code', cause=e)
2952         if self.get_param('youtube_print_sig_code'):
2953             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2954
2955         try:
2956             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
2957             ret = extract_nsig(jsi, func_code)(s)
2958         except JSInterpreter.Exception as e:
2959             try:
2960                 jsi = PhantomJSwrapper(self, timeout=5000)
2961             except ExtractorError:
2962                 raise e
2963             self.report_warning(
2964                 f'Native nsig extraction failed: Trying with PhantomJS\n'
2965                 f'         n = {s} ; player = {player_url}', video_id)
2966             self.write_debug(e, only_once=True)
2967
2968             args, func_body = func_code
2969             ret = jsi.execute(
2970                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
2971                 video_id=video_id, note='Executing signature code').strip()
2972
2973         self.write_debug(f'Decrypted nsig {s} => {ret}')
2974         return ret
2975
2976     def _extract_n_function_name(self, jscode):
2977         funcname, idx = self._search_regex(
2978             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2979             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2980         if not idx:
2981             return funcname
2982
2983         return json.loads(js_to_json(self._search_regex(
2984             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
2985             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
2986
2987     def _extract_n_function_code(self, video_id, player_url):
2988         player_id = self._extract_player_info(player_url)
2989         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
2990         jscode = func_code or self._load_player(video_id, player_url)
2991         jsi = JSInterpreter(jscode)
2992
2993         if func_code:
2994             return jsi, player_id, func_code
2995
2996         func_name = self._extract_n_function_name(jscode)
2997
2998         # For redundancy
2999         func_code = self._search_regex(
3000             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
3001                      # NB: The end of the regex is intentionally kept strict
3002                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
3003             jscode, 'nsig function', group=('var', 'code'), default=None)
3004         if func_code:
3005             func_code = ([func_code[0]], func_code[1])
3006         else:
3007             self.write_debug('Extracting nsig function with jsinterp')
3008             func_code = jsi.extract_function_code(func_name)
3009
3010         self.cache.store('youtube-nsig', player_id, func_code)
3011         return jsi, player_id, func_code
3012
3013     def _extract_n_function_from_code(self, jsi, func_code):
3014         func = jsi.extract_function_from_code(*func_code)
3015
3016         def extract_nsig(s):
3017             try:
3018                 ret = func([s])
3019             except JSInterpreter.Exception:
3020                 raise
3021             except Exception as e:
3022                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
3023
3024             if ret.startswith('enhanced_except_'):
3025                 raise JSInterpreter.Exception('Signature function returned an exception')
3026             return ret
3027
3028         return extract_nsig
3029
3030     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
3031         """
3032         Extract signatureTimestamp (sts)
3033         Required to tell API what sig/player version is in use.
3034         """
3035         sts = None
3036         if isinstance(ytcfg, dict):
3037             sts = int_or_none(ytcfg.get('STS'))
3038
3039         if not sts:
3040             # Attempt to extract from player
3041             if player_url is None:
3042                 error_msg = 'Cannot extract signature timestamp without player_url.'
3043                 if fatal:
3044                     raise ExtractorError(error_msg)
3045                 self.report_warning(error_msg)
3046                 return
3047             code = self._load_player(video_id, player_url, fatal=fatal)
3048             if code:
3049                 sts = int_or_none(self._search_regex(
3050                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
3051                     'JS player signature timestamp', group='sts', fatal=fatal))
3052         return sts
3053
3054     def _mark_watched(self, video_id, player_responses):
3055         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
3056             label = 'fully ' if is_full else ''
3057             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
3058                             expected_type=url_or_none)
3059             if not url:
3060                 self.report_warning(f'Unable to mark {label}watched')
3061                 return
3062             parsed_url = urllib.parse.urlparse(url)
3063             qs = urllib.parse.parse_qs(parsed_url.query)
3064
3065             # cpn generation algorithm is reverse engineered from base.js.
3066             # In fact it works even with dummy cpn.
3067             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
3068             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
3069
3070             # # more consistent results setting it to right before the end
3071             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
3072
3073             qs.update({
3074                 'ver': ['2'],
3075                 'cpn': [cpn],
3076                 'cmt': video_length,
3077                 'el': 'detailpage',  # otherwise defaults to "shorts"
3078             })
3079
3080             if is_full:
3081                 # these seem to mark watchtime "history" in the real world
3082                 # they're required, so send in a single value
3083                 qs.update({
3084                     'st': 0,
3085                     'et': video_length,
3086                 })
3087
3088             url = urllib.parse.urlunparse(
3089                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
3090
3091             self._download_webpage(
3092                 url, video_id, f'Marking {label}watched',
3093                 'Unable to mark watched', fatal=False)
3094
3095     @classmethod
3096     def _extract_from_webpage(cls, url, webpage):
3097         # Invidious Instances
3098         # https://github.com/yt-dlp/yt-dlp/issues/195
3099         # https://github.com/iv-org/invidious/pull/1730
3100         mobj = re.search(
3101             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
3102             webpage)
3103         if mobj:
3104             yield cls.url_result(mobj.group('url'), cls)
3105             raise cls.StopExtraction()
3106
3107         yield from super()._extract_from_webpage(url, webpage)
3108
3109         # lazyYT YouTube embed
3110         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
3111             yield cls.url_result(unescapeHTML(id_), cls, id_)
3112
3113         # Wordpress "YouTube Video Importer" plugin
3114         for m in re.findall(r'''(?x)<div[^>]+
3115                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
3116                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
3117             yield cls.url_result(m[-1], cls, m[-1])
3118
3119     @classmethod
3120     def extract_id(cls, url):
3121         video_id = cls.get_temp_id(url)
3122         if not video_id:
3123             raise ExtractorError(f'Invalid URL: {url}')
3124         return video_id
3125
3126     def _extract_chapters_from_json(self, data, duration):
3127         chapter_list = traverse_obj(
3128             data, (
3129                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3130                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3131             ), expected_type=list)
3132
3133         return self._extract_chapters(
3134             chapter_list,
3135             chapter_time=lambda chapter: float_or_none(
3136                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3137             chapter_title=lambda chapter: traverse_obj(
3138                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3139             duration=duration)
3140
3141     def _extract_chapters_from_engagement_panel(self, data, duration):
3142         content_list = traverse_obj(
3143             data,
3144             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3145             expected_type=list, default=[])
3146         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3147         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3148
3149         return next(filter(None, (
3150             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3151                                    chapter_time, chapter_title, duration)
3152             for contents in content_list)), [])
3153
3154     def _extract_chapters_from_description(self, description, duration):
3155         duration_re = r'(?:\d+:)?\d{1,2}:\d{2}'
3156         sep_re = r'(?m)^\s*(%s)\b\W*\s(%s)\s*$'
3157         return self._extract_chapters(
3158             re.findall(sep_re % (duration_re, r'.+?'), description or ''),
3159             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
3160             duration=duration, strict=False) or self._extract_chapters(
3161             re.findall(sep_re % (r'.+?', duration_re), description or ''),
3162             chapter_time=lambda x: parse_duration(x[1]), chapter_title=lambda x: x[0],
3163             duration=duration, strict=False)
3164
3165     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
3166         if not duration:
3167             return
3168         chapter_list = [{
3169             'start_time': chapter_time(chapter),
3170             'title': chapter_title(chapter),
3171         } for chapter in chapter_list or []]
3172         if not strict:
3173             chapter_list.sort(key=lambda c: c['start_time'] or 0)
3174
3175         chapters = [{'start_time': 0}]
3176         for idx, chapter in enumerate(chapter_list):
3177             if chapter['start_time'] is None:
3178                 self.report_warning(f'Incomplete chapter {idx}')
3179             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
3180                 chapters.append(chapter)
3181             elif chapter not in chapters:
3182                 self.report_warning(
3183                     f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
3184         return chapters[1:]
3185
3186     def _extract_comment(self, comment_renderer, parent=None):
3187         comment_id = comment_renderer.get('commentId')
3188         if not comment_id:
3189             return
3190
3191         text = self._get_text(comment_renderer, 'contentText')
3192
3193         # Timestamp is an estimate calculated from the current time and time_text
3194         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3195         timestamp = self._parse_time_text(time_text)
3196
3197         author = self._get_text(comment_renderer, 'authorText')
3198         author_id = try_get(comment_renderer,
3199                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3200
3201         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3202                                                        lambda x: x['likeCount']), str)) or 0
3203         author_thumbnail = try_get(comment_renderer,
3204                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3205
3206         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3207         is_favorited = 'creatorHeart' in (try_get(
3208             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3209         return {
3210             'id': comment_id,
3211             'text': text,
3212             'timestamp': timestamp,
3213             'time_text': time_text,
3214             'like_count': votes,
3215             'is_favorited': is_favorited,
3216             'author': author,
3217             'author_id': author_id,
3218             'author_thumbnail': author_thumbnail,
3219             'author_is_uploader': author_is_uploader,
3220             'parent': parent or 'root'
3221         }
3222
3223     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3224
3225         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3226
3227         def extract_header(contents):
3228             _continuation = None
3229             for content in contents:
3230                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3231                 expected_comment_count = self._get_count(
3232                     comments_header_renderer, 'countText', 'commentsCount')
3233
3234                 if expected_comment_count:
3235                     tracker['est_total'] = expected_comment_count
3236                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3237                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3238
3239                 sort_menu_item = try_get(
3240                     comments_header_renderer,
3241                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3242                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3243
3244                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3245                 if not _continuation:
3246                     continue
3247
3248                 sort_text = str_or_none(sort_menu_item.get('title'))
3249                 if not sort_text:
3250                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3251                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3252                 break
3253             return _continuation
3254
3255         def extract_thread(contents):
3256             if not parent:
3257                 tracker['current_page_thread'] = 0
3258             for content in contents:
3259                 if not parent and tracker['total_parent_comments'] >= max_parents:
3260                     yield
3261                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3262                 comment_renderer = get_first(
3263                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3264                     expected_type=dict, default={})
3265
3266                 comment = self._extract_comment(comment_renderer, parent)
3267                 if not comment:
3268                     continue
3269
3270                 tracker['running_total'] += 1
3271                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3272                 yield comment
3273
3274                 # Attempt to get the replies
3275                 comment_replies_renderer = try_get(
3276                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3277
3278                 if comment_replies_renderer:
3279                     tracker['current_page_thread'] += 1
3280                     comment_entries_iter = self._comment_entries(
3281                         comment_replies_renderer, ytcfg, video_id,
3282                         parent=comment.get('id'), tracker=tracker)
3283                     yield from itertools.islice(comment_entries_iter, min(
3284                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3285
3286         # Keeps track of counts across recursive calls
3287         if not tracker:
3288             tracker = dict(
3289                 running_total=0,
3290                 est_total=0,
3291                 current_page_thread=0,
3292                 total_parent_comments=0,
3293                 total_reply_comments=0)
3294
3295         # TODO: Deprecated
3296         # YouTube comments have a max depth of 2
3297         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3298         if max_depth:
3299             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3300                                                 'Set max replies in the max-comments extractor argument instead')
3301         if max_depth == 1 and parent:
3302             return
3303
3304         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3305             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3306
3307         continuation = self._extract_continuation(root_continuation_data)
3308
3309         response = None
3310         is_forced_continuation = False
3311         is_first_continuation = parent is None
3312         if is_first_continuation and not continuation:
3313             # Sometimes you can get comments by generating the continuation yourself,
3314             # even if YouTube initially reports them being disabled - e.g. stories comments.
3315             # Note: if the comment section is actually disabled, YouTube may return a response with
3316             # required check_get_keys missing. So we will disable that check initially in this case.
3317             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3318             is_forced_continuation = True
3319
3320         for page_num in itertools.count(0):
3321             if not continuation:
3322                 break
3323             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3324             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3325             if page_num == 0:
3326                 if is_first_continuation:
3327                     note_prefix = 'Downloading comment section API JSON'
3328                 else:
3329                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3330                         tracker['current_page_thread'], comment_prog_str)
3331             else:
3332                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3333                     '       ' if parent else '', ' replies' if parent else '',
3334                     page_num, comment_prog_str)
3335             try:
3336                 response = self._extract_response(
3337                     item_id=None, query=continuation,
3338                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3339                     check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3340             except ExtractorError as e:
3341                 # Ignore incomplete data error for replies if retries didn't work.
3342                 # This is to allow any other parent comments and comment threads to be downloaded.
3343                 # See: https://github.com/yt-dlp/yt-dlp/issues/4669
3344                 if 'incomplete data' in str(e).lower() and parent and self.get_param('ignoreerrors') is True:
3345                     self.report_warning(
3346                         'Received incomplete data for a comment reply thread and retrying did not help. '
3347                         'Ignoring to let other comments be downloaded.')
3348                 else:
3349                     raise
3350             is_forced_continuation = False
3351             continuation_contents = traverse_obj(
3352                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3353
3354             continuation = None
3355             for continuation_section in continuation_contents:
3356                 continuation_items = traverse_obj(
3357                     continuation_section,
3358                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3359                     get_all=False, expected_type=list) or []
3360                 if is_first_continuation:
3361                     continuation = extract_header(continuation_items)
3362                     is_first_continuation = False
3363                     if continuation:
3364                         break
3365                     continue
3366
3367                 for entry in extract_thread(continuation_items):
3368                     if not entry:
3369                         return
3370                     yield entry
3371                 continuation = self._extract_continuation({'contents': continuation_items})
3372                 if continuation:
3373                     break
3374
3375         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3376         if message and not parent and tracker['running_total'] == 0:
3377             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3378             raise self.CommentsDisabled
3379
3380     @staticmethod
3381     def _generate_comment_continuation(video_id):
3382         """
3383         Generates initial comment section continuation token from given video id
3384         """
3385         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3386         return base64.b64encode(token.encode()).decode()
3387
3388     def _get_comments(self, ytcfg, video_id, contents, webpage):
3389         """Entry for comment extraction"""
3390         def _real_comment_extract(contents):
3391             renderer = next((
3392                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3393                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3394             yield from self._comment_entries(renderer, ytcfg, video_id)
3395
3396         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3397         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3398
3399     @staticmethod
3400     def _get_checkok_params():
3401         return {'contentCheckOk': True, 'racyCheckOk': True}
3402
3403     @classmethod
3404     def _generate_player_context(cls, sts=None):
3405         context = {
3406             'html5Preference': 'HTML5_PREF_WANTS',
3407         }
3408         if sts is not None:
3409             context['signatureTimestamp'] = sts
3410         return {
3411             'playbackContext': {
3412                 'contentPlaybackContext': context
3413             },
3414             **cls._get_checkok_params()
3415         }
3416
3417     @staticmethod
3418     def _is_agegated(player_response):
3419         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3420             return True
3421
3422         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3423         AGE_GATE_REASONS = (
3424             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3425             'age_verification_required', 'age_check_required',  # status
3426         )
3427         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3428
3429     @staticmethod
3430     def _is_unplayable(player_response):
3431         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3432
3433     _STORY_PLAYER_PARAMS = '8AEB'
3434
3435     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3436
3437         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3438         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3439         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3440         headers = self.generate_api_headers(
3441             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3442
3443         yt_query = {
3444             'videoId': video_id,
3445         }
3446         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3447             yt_query['params'] = self._STORY_PLAYER_PARAMS
3448
3449         yt_query.update(self._generate_player_context(sts))
3450         return self._extract_response(
3451             item_id=video_id, ep='player', query=yt_query,
3452             ytcfg=player_ytcfg, headers=headers, fatal=True,
3453             default_client=client,
3454             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3455         ) or None
3456
3457     def _get_requested_clients(self, url, smuggled_data):
3458         requested_clients = []
3459         default = ['android', 'web']
3460         allowed_clients = sorted(
3461             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3462             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3463         for client in self._configuration_arg('player_client'):
3464             if client in allowed_clients:
3465                 requested_clients.append(client)
3466             elif client == 'default':
3467                 requested_clients.extend(default)
3468             elif client == 'all':
3469                 requested_clients.extend(allowed_clients)
3470             else:
3471                 self.report_warning(f'Skipping unsupported client {client}')
3472         if not requested_clients:
3473             requested_clients = default
3474
3475         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3476             requested_clients.extend(
3477                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3478
3479         return orderedSet(requested_clients)
3480
3481     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3482         initial_pr = None
3483         if webpage:
3484             initial_pr = self._search_json(
3485                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3486
3487         all_clients = set(clients)
3488         clients = clients[::-1]
3489         prs = []
3490
3491         def append_client(*client_names):
3492             """ Append the first client name that exists but not already used """
3493             for client_name in client_names:
3494                 actual_client = _split_innertube_client(client_name)[0]
3495                 if actual_client in INNERTUBE_CLIENTS:
3496                     if actual_client not in all_clients:
3497                         clients.append(client_name)
3498                         all_clients.add(actual_client)
3499                         return
3500
3501         # Android player_response does not have microFormats which are needed for
3502         # extraction of some data. So we return the initial_pr with formats
3503         # stripped out even if not requested by the user
3504         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3505         if initial_pr:
3506             pr = dict(initial_pr)
3507             pr['streamingData'] = None
3508             prs.append(pr)
3509
3510         last_error = None
3511         tried_iframe_fallback = False
3512         player_url = None
3513         while clients:
3514             client, base_client, variant = _split_innertube_client(clients.pop())
3515             player_ytcfg = master_ytcfg if client == 'web' else {}
3516             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3517                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3518
3519             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3520             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3521             if 'js' in self._configuration_arg('player_skip'):
3522                 require_js_player = False
3523                 player_url = None
3524
3525             if not player_url and not tried_iframe_fallback and require_js_player:
3526                 player_url = self._download_player_url(video_id)
3527                 tried_iframe_fallback = True
3528
3529             try:
3530                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3531                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3532             except ExtractorError as e:
3533                 if last_error:
3534                     self.report_warning(last_error)
3535                 last_error = e
3536                 continue
3537
3538             if pr:
3539                 # YouTube may return a different video player response than expected.
3540                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3541                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3542                 if pr_video_id and pr_video_id != video_id:
3543                     self.report_warning(
3544                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3545                 else:
3546                     prs.append(pr)
3547
3548             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3549             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3550                 append_client(f'{base_client}_creator')
3551             elif self._is_agegated(pr):
3552                 if variant == 'tv_embedded':
3553                     append_client(f'{base_client}_embedded')
3554                 elif not variant:
3555                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3556
3557         if last_error:
3558             if not len(prs):
3559                 raise last_error
3560             self.report_warning(last_error)
3561         return prs, player_url
3562
3563     def _needs_live_processing(self, live_status, duration):
3564         if (live_status == 'is_live' and self.get_param('live_from_start')
3565                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3566             return live_status
3567
3568     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3569         itags, stream_ids = collections.defaultdict(set), []
3570         itag_qualities, res_qualities = {}, {0: None}
3571         q = qualities([
3572             # Normally tiny is the smallest video-only formats. But
3573             # audio-only formats with unknown quality may get tagged as tiny
3574             'tiny',
3575             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3576             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3577         ])
3578         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3579
3580         for fmt in streaming_formats:
3581             if fmt.get('targetDurationSec'):
3582                 continue
3583
3584             itag = str_or_none(fmt.get('itag'))
3585             audio_track = fmt.get('audioTrack') or {}
3586             stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
3587             if stream_id in stream_ids:
3588                 continue
3589
3590             quality = fmt.get('quality')
3591             height = int_or_none(fmt.get('height'))
3592             if quality == 'tiny' or not quality:
3593                 quality = fmt.get('audioQuality', '').lower() or quality
3594             # The 3gp format (17) in android client has a quality of "small",
3595             # but is actually worse than other formats
3596             if itag == '17':
3597                 quality = 'tiny'
3598             if quality:
3599                 if itag:
3600                     itag_qualities[itag] = quality
3601                 if height:
3602                     res_qualities[height] = quality
3603             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3604             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3605             # number of fragment that would subsequently requested with (`&sq=N`)
3606             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3607                 continue
3608
3609             fmt_url = fmt.get('url')
3610             if not fmt_url:
3611                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3612                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3613                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3614                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3615                     continue
3616                 try:
3617                     fmt_url += '&%s=%s' % (
3618                         traverse_obj(sc, ('sp', -1)) or 'signature',
3619                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3620                     )
3621                 except ExtractorError as e:
3622                     self.report_warning('Signature extraction failed: Some formats may be missing',
3623                                         video_id=video_id, only_once=True)
3624                     self.write_debug(e, only_once=True)
3625                     continue
3626
3627             query = parse_qs(fmt_url)
3628             throttled = False
3629             if query.get('n'):
3630                 try:
3631                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3632                     fmt_url = update_url_query(fmt_url, {
3633                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3634                     })
3635                 except ExtractorError as e:
3636                     phantomjs_hint = ''
3637                     if isinstance(e, JSInterpreter.Exception):
3638                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3639                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3640                     if player_url:
3641                         self.report_warning(
3642                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3643                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3644                         self.write_debug(e, only_once=True)
3645                     else:
3646                         self.report_warning(
3647                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3648                             video_id=video_id, only_once=True)
3649                     throttled = True
3650
3651             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3652             language_preference = (
3653                 10 if audio_track.get('audioIsDefault') and 10
3654                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3655                 else -1)
3656             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3657             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3658             # Make sure to avoid false positives with small duration differences.
3659             # E.g. __2ABJjxzNo, ySuUZEjARPY
3660             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3661             if is_damaged:
3662                 self.report_warning(
3663                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3664             dct = {
3665                 'asr': int_or_none(fmt.get('audioSampleRate')),
3666                 'filesize': int_or_none(fmt.get('contentLength')),
3667                 'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
3668                 'format_note': join_nonempty(
3669                     '%s%s' % (audio_track.get('displayName') or '',
3670                               ' (default)' if language_preference > 0 else ''),
3671                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3672                     'DRC' if fmt.get('isDrc') else None,
3673                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3674                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3675                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3676                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3677                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3678                 'fps': int_or_none(fmt.get('fps')) or None,
3679                 'audio_channels': fmt.get('audioChannels'),
3680                 'height': height,
3681                 'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
3682                 'has_drm': bool(fmt.get('drmFamilies')),
3683                 'tbr': tbr,
3684                 'url': fmt_url,
3685                 'width': int_or_none(fmt.get('width')),
3686                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3687                                           'desc' if language_preference < -1 else '') or None,
3688                 'language_preference': language_preference,
3689                 # Strictly de-prioritize damaged and 3gp formats
3690                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3691             }
3692             mime_mobj = re.match(
3693                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3694             if mime_mobj:
3695                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3696                 dct.update(parse_codecs(mime_mobj.group(2)))
3697             no_audio = dct.get('acodec') == 'none'
3698             no_video = dct.get('vcodec') == 'none'
3699             if no_audio:
3700                 dct['vbr'] = tbr
3701             if no_video:
3702                 dct['abr'] = tbr
3703             if no_audio or no_video:
3704                 dct['downloader_options'] = {
3705                     # Youtube throttles chunks >~10M
3706                     'http_chunk_size': 10485760,
3707                 }
3708                 if dct.get('ext'):
3709                     dct['container'] = dct['ext'] + '_dash'
3710
3711             if itag:
3712                 itags[itag].add(('https', dct.get('language')))
3713                 stream_ids.append(stream_id)
3714             yield dct
3715
3716         needs_live_processing = self._needs_live_processing(live_status, duration)
3717         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3718
3719         skip_manifests = set(self._configuration_arg('skip'))
3720         if (not self.get_param('youtube_include_hls_manifest', True)
3721                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3722                 or needs_live_processing and skip_bad_formats):
3723             skip_manifests.add('hls')
3724
3725         if not self.get_param('youtube_include_dash_manifest', True):
3726             skip_manifests.add('dash')
3727         if self._configuration_arg('include_live_dash'):
3728             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3729                                                 'Use include_incomplete_formats extractor argument instead')
3730         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3731             skip_manifests.add('dash')
3732
3733         def process_manifest_format(f, proto, itag):
3734             key = (proto, f.get('language'))
3735             if key in itags[itag]:
3736                 return False
3737             itags[itag].add(key)
3738
3739             if any(p != proto for p, _ in itags[itag]):
3740                 f['format_id'] = f'{itag}-{proto}'
3741             elif itag:
3742                 f['format_id'] = itag
3743
3744             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3745             if f['quality'] == -1 and f.get('height'):
3746                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3747             return True
3748
3749         subtitles = {}
3750         for sd in streaming_data:
3751             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3752             if hls_manifest_url:
3753                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3754                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3755                 subtitles = self._merge_subtitles(subs, subtitles)
3756                 for f in fmts:
3757                     if process_manifest_format(f, 'hls', self._search_regex(
3758                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3759                         yield f
3760
3761             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3762             if dash_manifest_url:
3763                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3764                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3765                 for f in formats:
3766                     if process_manifest_format(f, 'dash', f['format_id']):
3767                         f['filesize'] = int_or_none(self._search_regex(
3768                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3769                         if needs_live_processing:
3770                             f['is_from_start'] = True
3771
3772                         yield f
3773         yield subtitles
3774
3775     def _extract_storyboard(self, player_responses, duration):
3776         spec = get_first(
3777             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3778         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3779         if not base_url:
3780             return
3781         L = len(spec) - 1
3782         for i, args in enumerate(spec):
3783             args = args.split('#')
3784             counts = list(map(int_or_none, args[:5]))
3785             if len(args) != 8 or not all(counts):
3786                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3787                 continue
3788             width, height, frame_count, cols, rows = counts
3789             N, sigh = args[6:]
3790
3791             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3792             fragment_count = frame_count / (cols * rows)
3793             fragment_duration = duration / fragment_count
3794             yield {
3795                 'format_id': f'sb{i}',
3796                 'format_note': 'storyboard',
3797                 'ext': 'mhtml',
3798                 'protocol': 'mhtml',
3799                 'acodec': 'none',
3800                 'vcodec': 'none',
3801                 'url': url,
3802                 'width': width,
3803                 'height': height,
3804                 'fps': frame_count / duration,
3805                 'rows': rows,
3806                 'columns': cols,
3807                 'fragments': [{
3808                     'url': url.replace('$M', str(j)),
3809                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3810                 } for j in range(math.ceil(fragment_count))],
3811             }
3812
3813     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3814         webpage = None
3815         if 'webpage' not in self._configuration_arg('player_skip'):
3816             query = {'bpctr': '9999999999', 'has_verified': '1'}
3817             if smuggled_data.get('is_story'):
3818                 query['pp'] = self._STORY_PLAYER_PARAMS
3819             webpage = self._download_webpage(
3820                 webpage_url, video_id, fatal=False, query=query)
3821
3822         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3823
3824         player_responses, player_url = self._extract_player_responses(
3825             self._get_requested_clients(url, smuggled_data),
3826             video_id, webpage, master_ytcfg, smuggled_data)
3827
3828         return webpage, master_ytcfg, player_responses, player_url
3829
3830     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3831         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3832         is_live = get_first(video_details, 'isLive')
3833         if is_live is None:
3834             is_live = get_first(live_broadcast_details, 'isLiveNow')
3835         live_content = get_first(video_details, 'isLiveContent')
3836         is_upcoming = get_first(video_details, 'isUpcoming')
3837         post_live = get_first(video_details, 'isPostLiveDvr')
3838         live_status = ('post_live' if post_live
3839                        else 'is_live' if is_live
3840                        else 'is_upcoming' if is_upcoming
3841                        else 'was_live' if live_content
3842                        else 'not_live' if False in (is_live, live_content)
3843                        else None)
3844         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3845         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3846
3847         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3848
3849     def _real_extract(self, url):
3850         url, smuggled_data = unsmuggle_url(url, {})
3851         video_id = self._match_id(url)
3852
3853         base_url = self.http_scheme() + '//www.youtube.com/'
3854         webpage_url = base_url + 'watch?v=' + video_id
3855
3856         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3857
3858         playability_statuses = traverse_obj(
3859             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3860
3861         trailer_video_id = get_first(
3862             playability_statuses,
3863             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3864             expected_type=str)
3865         if trailer_video_id:
3866             return self.url_result(
3867                 trailer_video_id, self.ie_key(), trailer_video_id)
3868
3869         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3870                        if webpage else (lambda x: None))
3871
3872         video_details = traverse_obj(
3873             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3874         microformats = traverse_obj(
3875             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3876             expected_type=dict, default=[])
3877
3878         translated_title = self._get_text(microformats, (..., 'title'))
3879         video_title = (self._preferred_lang and translated_title
3880                        or get_first(video_details, 'title')  # primary
3881                        or translated_title
3882                        or search_meta(['og:title', 'twitter:title', 'title']))
3883         translated_description = self._get_text(microformats, (..., 'description'))
3884         original_description = get_first(video_details, 'shortDescription')
3885         video_description = (
3886             self._preferred_lang and translated_description
3887             # If original description is blank, it will be an empty string.
3888             # Do not prefer translated description in this case.
3889             or original_description if original_description is not None else translated_description)
3890
3891         multifeed_metadata_list = get_first(
3892             player_responses,
3893             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3894             expected_type=str)
3895         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3896             if self.get_param('noplaylist'):
3897                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3898             else:
3899                 entries = []
3900                 feed_ids = []
3901                 for feed in multifeed_metadata_list.split(','):
3902                     # Unquote should take place before split on comma (,) since textual
3903                     # fields may contain comma as well (see
3904                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3905                     feed_data = urllib.parse.parse_qs(
3906                         urllib.parse.unquote_plus(feed))
3907
3908                     def feed_entry(name):
3909                         return try_get(
3910                             feed_data, lambda x: x[name][0], str)
3911
3912                     feed_id = feed_entry('id')
3913                     if not feed_id:
3914                         continue
3915                     feed_title = feed_entry('title')
3916                     title = video_title
3917                     if feed_title:
3918                         title += ' (%s)' % feed_title
3919                     entries.append({
3920                         '_type': 'url_transparent',
3921                         'ie_key': 'Youtube',
3922                         'url': smuggle_url(
3923                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3924                             {'force_singlefeed': True}),
3925                         'title': title,
3926                     })
3927                     feed_ids.append(feed_id)
3928                 self.to_screen(
3929                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3930                     % (', '.join(feed_ids), video_id))
3931                 return self.playlist_result(
3932                     entries, video_id, video_title, video_description)
3933
3934         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
3935                     or int_or_none(get_first(microformats, 'lengthSeconds'))
3936                     or parse_duration(search_meta('duration')) or None)
3937
3938         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
3939             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
3940         if live_status == 'post_live':
3941             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
3942
3943         if not formats:
3944             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3945                 self.report_drm(video_id)
3946             pemr = get_first(
3947                 playability_statuses,
3948                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3949             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3950             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3951             if subreason:
3952                 if subreason == 'The uploader has not made this video available in your country.':
3953                     countries = get_first(microformats, 'availableCountries')
3954                     if not countries:
3955                         regions_allowed = search_meta('regionsAllowed')
3956                         countries = regions_allowed.split(',') if regions_allowed else None
3957                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3958                 reason += f'. {subreason}'
3959             if reason:
3960                 self.raise_no_formats(reason, expected=True)
3961
3962         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3963         if not keywords and webpage:
3964             keywords = [
3965                 unescapeHTML(m.group('content'))
3966                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3967         for keyword in keywords:
3968             if keyword.startswith('yt:stretch='):
3969                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3970                 if mobj:
3971                     # NB: float is intentional for forcing float division
3972                     w, h = (float(v) for v in mobj.groups())
3973                     if w > 0 and h > 0:
3974                         ratio = w / h
3975                         for f in formats:
3976                             if f.get('vcodec') != 'none':
3977                                 f['stretched_ratio'] = ratio
3978                         break
3979         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3980         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3981         if thumbnail_url:
3982             thumbnails.append({
3983                 'url': thumbnail_url,
3984             })
3985         original_thumbnails = thumbnails.copy()
3986
3987         # The best resolution thumbnails sometimes does not appear in the webpage
3988         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3989         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3990         thumbnail_names = [
3991             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3992             # in resolution, these are not the custom thumbnail. So de-prioritize them
3993             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3994             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3995         ]
3996         n_thumbnail_names = len(thumbnail_names)
3997         thumbnails.extend({
3998             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3999                 video_id=video_id, name=name, ext=ext,
4000                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
4001         } for name in thumbnail_names for ext in ('webp', 'jpg'))
4002         for thumb in thumbnails:
4003             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
4004             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
4005         self._remove_duplicate_formats(thumbnails)
4006         self._downloader._sort_thumbnails(original_thumbnails)
4007
4008         category = get_first(microformats, 'category') or search_meta('genre')
4009         channel_id = str_or_none(
4010             get_first(video_details, 'channelId')
4011             or get_first(microformats, 'externalChannelId')
4012             or search_meta('channelId'))
4013         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
4014
4015         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
4016         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
4017         if not duration and live_end_time and live_start_time:
4018             duration = live_end_time - live_start_time
4019
4020         needs_live_processing = self._needs_live_processing(live_status, duration)
4021
4022         def is_bad_format(fmt):
4023             if needs_live_processing and not fmt.get('is_from_start'):
4024                 return True
4025             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
4026                     and fmt.get('protocol') == 'http_dash_segments'):
4027                 return True
4028
4029         for fmt in filter(is_bad_format, formats):
4030             fmt['preference'] = (fmt.get('preference') or -1) - 10
4031             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
4032
4033         if needs_live_processing:
4034             self._prepare_live_from_start_formats(
4035                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
4036
4037         formats.extend(self._extract_storyboard(player_responses, duration))
4038
4039         info = {
4040             'id': video_id,
4041             'title': video_title,
4042             'formats': formats,
4043             'thumbnails': thumbnails,
4044             # The best thumbnail that we are sure exists. Prevents unnecessary
4045             # URL checking if user don't care about getting the best possible thumbnail
4046             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
4047             'description': video_description,
4048             'uploader': get_first(video_details, 'author'),
4049             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
4050             'uploader_url': owner_profile_url,
4051             'channel_id': channel_id,
4052             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
4053             'duration': duration,
4054             'view_count': int_or_none(
4055                 get_first((video_details, microformats), (..., 'viewCount'))
4056                 or search_meta('interactionCount')),
4057             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
4058             'age_limit': 18 if (
4059                 get_first(microformats, 'isFamilySafe') is False
4060                 or search_meta('isFamilyFriendly') == 'false'
4061                 or search_meta('og:restrictions:age') == '18+') else 0,
4062             'webpage_url': webpage_url,
4063             'categories': [category] if category else None,
4064             'tags': keywords,
4065             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
4066             'live_status': live_status,
4067             'release_timestamp': live_start_time,
4068             '_format_sort_fields': (  # source_preference is lower for throttled/potentially damaged formats
4069                 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto')
4070         }
4071
4072         subtitles = {}
4073         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
4074         if pctr:
4075             def get_lang_code(track):
4076                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
4077                         or track.get('languageCode'))
4078
4079             # Converted into dicts to remove duplicates
4080             captions = {
4081                 get_lang_code(sub): sub
4082                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
4083             translation_languages = {
4084                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
4085                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
4086
4087             def process_language(container, base_url, lang_code, sub_name, query):
4088                 lang_subs = container.setdefault(lang_code, [])
4089                 for fmt in self._SUBTITLE_FORMATS:
4090                     query.update({
4091                         'fmt': fmt,
4092                     })
4093                     lang_subs.append({
4094                         'ext': fmt,
4095                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
4096                         'name': sub_name,
4097                     })
4098
4099             # NB: Constructing the full subtitle dictionary is slow
4100             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
4101                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
4102             for lang_code, caption_track in captions.items():
4103                 base_url = caption_track.get('baseUrl')
4104                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
4105                 if not base_url:
4106                     continue
4107                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
4108                 if caption_track.get('kind') != 'asr':
4109                     if not lang_code:
4110                         continue
4111                     process_language(
4112                         subtitles, base_url, lang_code, lang_name, {})
4113                     if not caption_track.get('isTranslatable'):
4114                         continue
4115                 for trans_code, trans_name in translation_languages.items():
4116                     if not trans_code:
4117                         continue
4118                     orig_trans_code = trans_code
4119                     if caption_track.get('kind') != 'asr' and trans_code != 'und':
4120                         if not get_translated_subs:
4121                             continue
4122                         trans_code += f'-{lang_code}'
4123                         trans_name += format_field(lang_name, None, ' from %s')
4124                     # Add an "-orig" label to the original language so that it can be distinguished.
4125                     # The subs are returned without "-orig" as well for compatibility
4126                     if lang_code == f'a-{orig_trans_code}':
4127                         process_language(
4128                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
4129                     # Setting tlang=lang returns damaged subtitles.
4130                     process_language(automatic_captions, base_url, trans_code, trans_name,
4131                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
4132
4133         info['automatic_captions'] = automatic_captions
4134         info['subtitles'] = subtitles
4135
4136         parsed_url = urllib.parse.urlparse(url)
4137         for component in [parsed_url.fragment, parsed_url.query]:
4138             query = urllib.parse.parse_qs(component)
4139             for k, v in query.items():
4140                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4141                     d_k += '_time'
4142                     if d_k not in info and k in s_ks:
4143                         info[d_k] = parse_duration(query[k][0])
4144
4145         # Youtube Music Auto-generated description
4146         if video_description:
4147             mobj = re.search(
4148                 r'''(?xs)
4149                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4150                     (?P<album>[^\n]+)
4151                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4152                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4153                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4154                     .+\nAuto-generated\ by\ YouTube\.\s*$
4155                 ''', video_description)
4156             if mobj:
4157                 release_year = mobj.group('release_year')
4158                 release_date = mobj.group('release_date')
4159                 if release_date:
4160                     release_date = release_date.replace('-', '')
4161                     if not release_year:
4162                         release_year = release_date[:4]
4163                 info.update({
4164                     'album': mobj.group('album'.strip()),
4165                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4166                     'track': mobj.group('track').strip(),
4167                     'release_date': release_date,
4168                     'release_year': int_or_none(release_year),
4169                 })
4170
4171         initial_data = None
4172         if webpage:
4173             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4174         if not initial_data:
4175             query = {'videoId': video_id}
4176             query.update(self._get_checkok_params())
4177             initial_data = self._extract_response(
4178                 item_id=video_id, ep='next', fatal=False,
4179                 ytcfg=master_ytcfg, query=query,
4180                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4181                 note='Downloading initial data API JSON')
4182
4183         info['comment_count'] = traverse_obj(initial_data, (
4184             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4185             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
4186         ), (
4187             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4188             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
4189         ), expected_type=int_or_none, get_all=False)
4190
4191         try:  # This will error if there is no livechat
4192             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4193         except (KeyError, IndexError, TypeError):
4194             pass
4195         else:
4196             info.setdefault('subtitles', {})['live_chat'] = [{
4197                 # url is needed to set cookies
4198                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4199                 'video_id': video_id,
4200                 'ext': 'json',
4201                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4202                              else 'youtube_live_chat_replay'),
4203             }]
4204
4205         if initial_data:
4206             info['chapters'] = (
4207                 self._extract_chapters_from_json(initial_data, duration)
4208                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4209                 or self._extract_chapters_from_description(video_description, duration)
4210                 or None)
4211
4212         contents = traverse_obj(
4213             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4214             expected_type=list, default=[])
4215
4216         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4217         if vpir:
4218             stl = vpir.get('superTitleLink')
4219             if stl:
4220                 stl = self._get_text(stl)
4221                 if try_get(
4222                         vpir,
4223                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4224                     info['location'] = stl
4225                 else:
4226                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4227                     if mobj:
4228                         info.update({
4229                             'series': mobj.group(1),
4230                             'season_number': int(mobj.group(2)),
4231                             'episode_number': int(mobj.group(3)),
4232                         })
4233             for tlb in (try_get(
4234                     vpir,
4235                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4236                     list) or []):
4237                 tbrs = variadic(
4238                     traverse_obj(
4239                         tlb, 'toggleButtonRenderer',
4240                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer'),
4241                         default=[]))
4242                 for tbr in tbrs:
4243                     for getter, regex in [(
4244                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4245                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4246                                 lambda x: x['accessibility'],
4247                                 lambda x: x['accessibilityData']['accessibilityData'],
4248                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4249                         label = (try_get(tbr, getter, dict) or {}).get('label')
4250                         if label:
4251                             mobj = re.match(regex, label)
4252                             if mobj:
4253                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4254                                 break
4255             sbr_tooltip = try_get(
4256                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4257             if sbr_tooltip:
4258                 like_count, dislike_count = sbr_tooltip.split(' / ')
4259                 info.update({
4260                     'like_count': str_to_int(like_count),
4261                     'dislike_count': str_to_int(dislike_count),
4262                 })
4263             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4264             if vcr:
4265                 vc = self._get_count(vcr, 'viewCount')
4266                 # Upcoming premieres with waiting count are treated as live here
4267                 if vcr.get('isLive'):
4268                     info['concurrent_view_count'] = vc
4269                 elif info.get('view_count') is None:
4270                     info['view_count'] = vc
4271
4272         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4273         if vsir:
4274             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4275             info.update({
4276                 'channel': self._get_text(vor, 'title'),
4277                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4278
4279             rows = try_get(
4280                 vsir,
4281                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4282                 list) or []
4283             multiple_songs = False
4284             for row in rows:
4285                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4286                     multiple_songs = True
4287                     break
4288             for row in rows:
4289                 mrr = row.get('metadataRowRenderer') or {}
4290                 mrr_title = mrr.get('title')
4291                 if not mrr_title:
4292                     continue
4293                 mrr_title = self._get_text(mrr, 'title')
4294                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4295                 if mrr_title == 'License':
4296                     info['license'] = mrr_contents_text
4297                 elif not multiple_songs:
4298                     if mrr_title == 'Album':
4299                         info['album'] = mrr_contents_text
4300                     elif mrr_title == 'Artist':
4301                         info['artist'] = mrr_contents_text
4302                     elif mrr_title == 'Song':
4303                         info['track'] = mrr_contents_text
4304
4305         fallbacks = {
4306             'channel': 'uploader',
4307             'channel_id': 'uploader_id',
4308             'channel_url': 'uploader_url',
4309         }
4310
4311         # The upload date for scheduled, live and past live streams / premieres in microformats
4312         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4313         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4314         upload_date = (
4315             unified_strdate(get_first(microformats, 'uploadDate'))
4316             or unified_strdate(search_meta('uploadDate')))
4317         if not upload_date or (
4318             live_status in ('not_live', None)
4319             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4320         ):
4321             upload_date = strftime_or_none(
4322                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4323         info['upload_date'] = upload_date
4324
4325         for to, frm in fallbacks.items():
4326             if not info.get(to):
4327                 info[to] = info.get(frm)
4328
4329         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4330             v = info.get(s_k)
4331             if v:
4332                 info[d_k] = v
4333
4334         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4335
4336         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4337                       or get_first(video_details, 'isPrivate', expected_type=bool))
4338
4339         info['availability'] = (
4340             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4341             else self._availability(
4342                 is_private=is_private,
4343                 needs_premium=(
4344                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4345                     or False if initial_data and is_private is not None else None),
4346                 needs_subscription=(
4347                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4348                     or False if initial_data and is_private is not None else None),
4349                 needs_auth=info['age_limit'] >= 18,
4350                 is_unlisted=None if is_private is None else (
4351                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4352                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4353
4354         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4355
4356         self.mark_watched(video_id, player_responses)
4357
4358         return info
4359
4360
4361 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4362     @staticmethod
4363     def passthrough_smuggled_data(func):
4364         def _smuggle(info, smuggled_data):
4365             if info.get('_type') not in ('url', 'url_transparent'):
4366                 return info
4367             if smuggled_data.get('is_music_url'):
4368                 parsed_url = urllib.parse.urlparse(info['url'])
4369                 if parsed_url.netloc in ('www.youtube.com', 'music.youtube.com'):
4370                     smuggled_data.pop('is_music_url')
4371                     info['url'] = urllib.parse.urlunparse(parsed_url._replace(netloc='music.youtube.com'))
4372             if smuggled_data:
4373                 info['url'] = smuggle_url(info['url'], smuggled_data)
4374             return info
4375
4376         @functools.wraps(func)
4377         def wrapper(self, url):
4378             url, smuggled_data = unsmuggle_url(url, {})
4379             if self.is_music_url(url):
4380                 smuggled_data['is_music_url'] = True
4381             info_dict = func(self, url, smuggled_data)
4382             if smuggled_data:
4383                 _smuggle(info_dict, smuggled_data)
4384                 if info_dict.get('entries'):
4385                     info_dict['entries'] = (_smuggle(i, smuggled_data.copy()) for i in info_dict['entries'])
4386             return info_dict
4387         return wrapper
4388
4389     def _extract_channel_id(self, webpage):
4390         channel_id = self._html_search_meta(
4391             'channelId', webpage, 'channel id', default=None)
4392         if channel_id:
4393             return channel_id
4394         channel_url = self._html_search_meta(
4395             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
4396              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
4397              'twitter:app:url:googleplay'), webpage, 'channel url')
4398         return self._search_regex(
4399             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
4400             channel_url, 'channel id')
4401
4402     @staticmethod
4403     def _extract_basic_item_renderer(item):
4404         # Modified from _extract_grid_item_renderer
4405         known_basic_renderers = (
4406             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4407         )
4408         for key, renderer in item.items():
4409             if not isinstance(renderer, dict):
4410                 continue
4411             elif key in known_basic_renderers:
4412                 return renderer
4413             elif key.startswith('grid') and key.endswith('Renderer'):
4414                 return renderer
4415
4416     def _extract_channel_renderer(self, renderer):
4417         channel_id = renderer['channelId']
4418         title = self._get_text(renderer, 'title')
4419         channel_url = f'https://www.youtube.com/channel/{channel_id}'
4420         return {
4421             '_type': 'url',
4422             'url': channel_url,
4423             'id': channel_id,
4424             'ie_key': YoutubeTabIE.ie_key(),
4425             'channel': title,
4426             'channel_id': channel_id,
4427             'channel_url': channel_url,
4428             'title': title,
4429             'channel_follower_count': self._get_count(renderer, 'subscriberCountText'),
4430             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
4431             'playlist_count': self._get_count(renderer, 'videoCountText'),
4432             'description': self._get_text(renderer, 'descriptionSnippet'),
4433         }
4434
4435     def _grid_entries(self, grid_renderer):
4436         for item in grid_renderer['items']:
4437             if not isinstance(item, dict):
4438                 continue
4439             renderer = self._extract_basic_item_renderer(item)
4440             if not isinstance(renderer, dict):
4441                 continue
4442             title = self._get_text(renderer, 'title')
4443
4444             # playlist
4445             playlist_id = renderer.get('playlistId')
4446             if playlist_id:
4447                 yield self.url_result(
4448                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4449                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4450                     video_title=title)
4451                 continue
4452             # video
4453             video_id = renderer.get('videoId')
4454             if video_id:
4455                 yield self._extract_video(renderer)
4456                 continue
4457             # channel
4458             channel_id = renderer.get('channelId')
4459             if channel_id:
4460                 yield self._extract_channel_renderer(renderer)
4461                 continue
4462             # generic endpoint URL support
4463             ep_url = urljoin('https://www.youtube.com/', try_get(
4464                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4465                 str))
4466             if ep_url:
4467                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4468                     if ie.suitable(ep_url):
4469                         yield self.url_result(
4470                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4471                         break
4472
4473     def _music_reponsive_list_entry(self, renderer):
4474         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4475         if video_id:
4476             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4477                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4478         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4479         if playlist_id:
4480             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4481             if video_id:
4482                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4483                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4484             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4485                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4486         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4487         if browse_id:
4488             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4489                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4490
4491     def _shelf_entries_from_content(self, shelf_renderer):
4492         content = shelf_renderer.get('content')
4493         if not isinstance(content, dict):
4494             return
4495         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4496         if renderer:
4497             # TODO: add support for nested playlists so each shelf is processed
4498             # as separate playlist
4499             # TODO: this includes only first N items
4500             yield from self._grid_entries(renderer)
4501         renderer = content.get('horizontalListRenderer')
4502         if renderer:
4503             # TODO
4504             pass
4505
4506     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4507         ep = try_get(
4508             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4509             str)
4510         shelf_url = urljoin('https://www.youtube.com', ep)
4511         if shelf_url:
4512             # Skipping links to another channels, note that checking for
4513             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4514             # will not work
4515             if skip_channels and '/channels?' in shelf_url:
4516                 return
4517             title = self._get_text(shelf_renderer, 'title')
4518             yield self.url_result(shelf_url, video_title=title)
4519         # Shelf may not contain shelf URL, fallback to extraction from content
4520         yield from self._shelf_entries_from_content(shelf_renderer)
4521
4522     def _playlist_entries(self, video_list_renderer):
4523         for content in video_list_renderer['contents']:
4524             if not isinstance(content, dict):
4525                 continue
4526             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4527             if not isinstance(renderer, dict):
4528                 continue
4529             video_id = renderer.get('videoId')
4530             if not video_id:
4531                 continue
4532             yield self._extract_video(renderer)
4533
4534     def _rich_entries(self, rich_grid_renderer):
4535         renderer = traverse_obj(
4536             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4537         video_id = renderer.get('videoId')
4538         if not video_id:
4539             return
4540         yield self._extract_video(renderer)
4541
4542     def _video_entry(self, video_renderer):
4543         video_id = video_renderer.get('videoId')
4544         if video_id:
4545             return self._extract_video(video_renderer)
4546
4547     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4548         url = urljoin('https://youtube.com', traverse_obj(
4549             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4550         if url:
4551             return self.url_result(
4552                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4553
4554     def _post_thread_entries(self, post_thread_renderer):
4555         post_renderer = try_get(
4556             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4557         if not post_renderer:
4558             return
4559         # video attachment
4560         video_renderer = try_get(
4561             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4562         video_id = video_renderer.get('videoId')
4563         if video_id:
4564             entry = self._extract_video(video_renderer)
4565             if entry:
4566                 yield entry
4567         # playlist attachment
4568         playlist_id = try_get(
4569             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4570         if playlist_id:
4571             yield self.url_result(
4572                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4573                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4574         # inline video links
4575         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4576         for run in runs:
4577             if not isinstance(run, dict):
4578                 continue
4579             ep_url = try_get(
4580                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4581             if not ep_url:
4582                 continue
4583             if not YoutubeIE.suitable(ep_url):
4584                 continue
4585             ep_video_id = YoutubeIE._match_id(ep_url)
4586             if video_id == ep_video_id:
4587                 continue
4588             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4589
4590     def _post_thread_continuation_entries(self, post_thread_continuation):
4591         contents = post_thread_continuation.get('contents')
4592         if not isinstance(contents, list):
4593             return
4594         for content in contents:
4595             renderer = content.get('backstagePostThreadRenderer')
4596             if isinstance(renderer, dict):
4597                 yield from self._post_thread_entries(renderer)
4598                 continue
4599             renderer = content.get('videoRenderer')
4600             if isinstance(renderer, dict):
4601                 yield self._video_entry(renderer)
4602
4603     r''' # unused
4604     def _rich_grid_entries(self, contents):
4605         for content in contents:
4606             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4607             if video_renderer:
4608                 entry = self._video_entry(video_renderer)
4609                 if entry:
4610                     yield entry
4611     '''
4612
4613     def _report_history_entries(self, renderer):
4614         for url in traverse_obj(renderer, (
4615                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4616                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4617                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4618             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4619
4620     def _extract_entries(self, parent_renderer, continuation_list):
4621         # continuation_list is modified in-place with continuation_list = [continuation_token]
4622         continuation_list[:] = [None]
4623         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4624         for content in contents:
4625             if not isinstance(content, dict):
4626                 continue
4627             is_renderer = traverse_obj(
4628                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4629                 expected_type=dict)
4630             if not is_renderer:
4631                 if content.get('richItemRenderer'):
4632                     for entry in self._rich_entries(content['richItemRenderer']):
4633                         yield entry
4634                     continuation_list[0] = self._extract_continuation(parent_renderer)
4635                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4636                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4637                     yield from self._report_history_entries(table)
4638                     continuation_list[0] = self._extract_continuation(table)
4639                 continue
4640
4641             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4642             for isr_content in isr_contents:
4643                 if not isinstance(isr_content, dict):
4644                     continue
4645
4646                 known_renderers = {
4647                     'playlistVideoListRenderer': self._playlist_entries,
4648                     'gridRenderer': self._grid_entries,
4649                     'reelShelfRenderer': self._grid_entries,
4650                     'shelfRenderer': self._shelf_entries,
4651                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4652                     'backstagePostThreadRenderer': self._post_thread_entries,
4653                     'videoRenderer': lambda x: [self._video_entry(x)],
4654                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4655                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4656                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4657                 }
4658                 for key, renderer in isr_content.items():
4659                     if key not in known_renderers:
4660                         continue
4661                     for entry in known_renderers[key](renderer):
4662                         if entry:
4663                             yield entry
4664                     continuation_list[0] = self._extract_continuation(renderer)
4665                     break
4666
4667             if not continuation_list[0]:
4668                 continuation_list[0] = self._extract_continuation(is_renderer)
4669
4670         if not continuation_list[0]:
4671             continuation_list[0] = self._extract_continuation(parent_renderer)
4672
4673     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4674         continuation_list = [None]
4675         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4676         tab_content = try_get(tab, lambda x: x['content'], dict)
4677         if not tab_content:
4678             return
4679         parent_renderer = (
4680             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4681             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4682         yield from extract_entries(parent_renderer)
4683         continuation = continuation_list[0]
4684
4685         for page_num in itertools.count(1):
4686             if not continuation:
4687                 break
4688             headers = self.generate_api_headers(
4689                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4690             response = self._extract_response(
4691                 item_id=f'{item_id} page {page_num}',
4692                 query=continuation, headers=headers, ytcfg=ytcfg,
4693                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4694
4695             if not response:
4696                 break
4697             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4698             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4699             visitor_data = self._extract_visitor_data(response) or visitor_data
4700
4701             known_renderers = {
4702                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4703                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4704                 'gridVideoRenderer': (self._grid_entries, 'items'),
4705                 'gridChannelRenderer': (self._grid_entries, 'items'),
4706                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4707                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4708                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4709                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4710                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4711                 'playlistVideoListContinuation': (self._playlist_entries, None),
4712                 'gridContinuation': (self._grid_entries, None),
4713                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4714                 'sectionListContinuation': (extract_entries, None),  # for feeds
4715             }
4716
4717             continuation_items = traverse_obj(response, (
4718                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4719                 'appendContinuationItemsAction', 'continuationItems'
4720             ), 'continuationContents', get_all=False)
4721             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4722
4723             video_items_renderer = None
4724             for key in continuation_item.keys():
4725                 if key not in known_renderers:
4726                     continue
4727                 func, parent_key = known_renderers[key]
4728                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4729                 continuation_list = [None]
4730                 yield from func(video_items_renderer)
4731                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4732
4733             if not video_items_renderer:
4734                 break
4735
4736     @staticmethod
4737     def _extract_selected_tab(tabs, fatal=True):
4738         for tab_renderer in tabs:
4739             if tab_renderer.get('selected'):
4740                 return tab_renderer
4741         if fatal:
4742             raise ExtractorError('Unable to find selected tab')
4743
4744     @staticmethod
4745     def _extract_tab_renderers(response):
4746         return traverse_obj(
4747             response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
4748
4749     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4750         metadata = self._extract_metadata_from_tabs(item_id, data)
4751
4752         selected_tab = self._extract_selected_tab(tabs)
4753         metadata['title'] += format_field(selected_tab, 'title', ' - %s')
4754         metadata['title'] += format_field(selected_tab, 'expandedText', ' - %s')
4755
4756         return self.playlist_result(
4757             self._entries(
4758                 selected_tab, metadata['id'], ytcfg,
4759                 self._extract_account_syncid(ytcfg, data),
4760                 self._extract_visitor_data(data, ytcfg)),
4761             **metadata)
4762
4763     def _extract_metadata_from_tabs(self, item_id, data):
4764         info = {'id': item_id}
4765
4766         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
4767         if metadata_renderer:
4768             info.update({
4769                 'uploader': metadata_renderer.get('title'),
4770                 'uploader_id': metadata_renderer.get('externalId'),
4771                 'uploader_url': metadata_renderer.get('channelUrl'),
4772             })
4773             if info['uploader_id']:
4774                 info['id'] = info['uploader_id']
4775         else:
4776             metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
4777
4778         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4779         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4780         def _get_uncropped(url):
4781             return url_or_none((url or '').split('=')[0] + '=s0')
4782
4783         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
4784         if avatar_thumbnails:
4785             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4786             if uncropped_avatar:
4787                 avatar_thumbnails.append({
4788                     'url': uncropped_avatar,
4789                     'id': 'avatar_uncropped',
4790                     'preference': 1
4791                 })
4792
4793         channel_banners = self._extract_thumbnails(
4794             data, ('header', ..., ('banner', 'mobileBanner', 'tvBanner')))
4795         for banner in channel_banners:
4796             banner['preference'] = -10
4797
4798         if channel_banners:
4799             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4800             if uncropped_banner:
4801                 channel_banners.append({
4802                     'url': uncropped_banner,
4803                     'id': 'banner_uncropped',
4804                     'preference': -5
4805                 })
4806
4807         # Deprecated - remove primary_sidebar_renderer when layout discontinued
4808         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4809         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
4810
4811         primary_thumbnails = self._extract_thumbnails(
4812             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4813         playlist_thumbnails = self._extract_thumbnails(
4814             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
4815
4816         info.update({
4817             'title': (traverse_obj(metadata_renderer, 'title')
4818                       or self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag'))
4819                       or info['id']),
4820             'availability': self._extract_availability(data),
4821             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4822             'description': try_get(metadata_renderer, lambda x: x.get('description', '')),
4823             'tags': try_get(metadata_renderer or {}, lambda x: x.get('keywords', '').split()),
4824             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
4825         })
4826
4827         # Playlist stats is a text runs array containing [video count, view count, last updated].
4828         # last updated or (view count and last updated) may be missing.
4829         playlist_stats = get_first(
4830             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'), ))
4831
4832         last_updated_unix = self._parse_time_text(
4833             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
4834             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
4835         info['modified_date'] = strftime_or_none(last_updated_unix, '%Y%m%d')
4836
4837         info['view_count'] = self._get_count(playlist_stats, 1)
4838         if info['view_count'] is None:  # 0 is allowed
4839             info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
4840
4841         info['playlist_count'] = self._get_count(playlist_stats, 0)
4842         if info['playlist_count'] is None:  # 0 is allowed
4843             info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
4844
4845         if not info.get('uploader_id'):
4846             owner = traverse_obj(playlist_header_renderer, 'ownerText')
4847             if not owner:  # Deprecated
4848                 owner = traverse_obj(
4849                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
4850                     ('videoOwner', 'videoOwnerRenderer', 'title'))
4851             owner_text = self._get_text(owner)
4852             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
4853             info.update({
4854                 'uploader': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
4855                 'uploader_id': browse_ep.get('browseId'),
4856                 'uploader_url': urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))
4857             })
4858
4859         info.update({
4860             'channel': info['uploader'],
4861             'channel_id': info['uploader_id'],
4862             'channel_url': info['uploader_url']
4863         })
4864         return info
4865
4866     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4867         first_id = last_id = response = None
4868         for page_num in itertools.count(1):
4869             videos = list(self._playlist_entries(playlist))
4870             if not videos:
4871                 return
4872             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4873             if start >= len(videos):
4874                 return
4875             yield from videos[start:]
4876             first_id = first_id or videos[0]['id']
4877             last_id = videos[-1]['id']
4878             watch_endpoint = try_get(
4879                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4880             headers = self.generate_api_headers(
4881                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4882                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4883             query = {
4884                 'playlistId': playlist_id,
4885                 'videoId': watch_endpoint.get('videoId') or last_id,
4886                 'index': watch_endpoint.get('index') or len(videos),
4887                 'params': watch_endpoint.get('params') or 'OAE%3D'
4888             }
4889             response = self._extract_response(
4890                 item_id='%s page %d' % (playlist_id, page_num),
4891                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4892                 check_get_keys='contents'
4893             )
4894             playlist = try_get(
4895                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4896
4897     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4898         title = playlist.get('title') or try_get(
4899             data, lambda x: x['titleText']['simpleText'], str)
4900         playlist_id = playlist.get('playlistId') or item_id
4901
4902         # Delegating everything except mix playlists to regular tab-based playlist URL
4903         playlist_url = urljoin(url, try_get(
4904             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4905             str))
4906
4907         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4908         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4909         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4910
4911         if playlist_url and playlist_url != url and not is_known_unviewable:
4912             return self.url_result(
4913                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4914                 video_title=title)
4915
4916         return self.playlist_result(
4917             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4918             playlist_id=playlist_id, playlist_title=title)
4919
4920     def _extract_availability(self, data):
4921         """
4922         Gets the availability of a given playlist/tab.
4923         Note: Unless YouTube tells us explicitly, we do not assume it is public
4924         @param data: response
4925         """
4926         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4927         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
4928         player_header_privacy = playlist_header_renderer.get('privacy')
4929
4930         badges = self._extract_badges(sidebar_renderer)
4931
4932         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4933         privacy_setting_icon = get_first(
4934             (playlist_header_renderer, sidebar_renderer),
4935             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
4936              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
4937             expected_type=str)
4938
4939         microformats_is_unlisted = traverse_obj(
4940             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
4941
4942         return (
4943             'public' if (
4944                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4945                 or player_header_privacy == 'PUBLIC'
4946                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
4947             else self._availability(
4948                 is_private=(
4949                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4950                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
4951                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
4952                 is_unlisted=(
4953                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4954                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
4955                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
4956                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
4957                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
4958                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
4959                 needs_auth=False))
4960
4961     @staticmethod
4962     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4963         sidebar_renderer = try_get(
4964             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4965         for item in sidebar_renderer:
4966             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4967             if renderer:
4968                 return renderer
4969
4970     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4971         """
4972         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
4973         """
4974         is_playlist = bool(traverse_obj(
4975             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
4976         if not is_playlist:
4977             return
4978         headers = self.generate_api_headers(
4979             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4980             visitor_data=self._extract_visitor_data(data, ytcfg))
4981         query = {
4982             'params': 'wgYCCAA=',
4983             'browseId': f'VL{item_id}'
4984         }
4985         return self._extract_response(
4986             item_id=item_id, headers=headers, query=query,
4987             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4988             note='Redownloading playlist API JSON with unavailable videos')
4989
4990     @functools.cached_property
4991     def skip_webpage(self):
4992         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4993
4994     def _extract_webpage(self, url, item_id, fatal=True):
4995         webpage, data = None, None
4996         for retry in self.RetryManager(fatal=fatal):
4997             try:
4998                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
4999                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
5000             except ExtractorError as e:
5001                 if isinstance(e.cause, network_exceptions):
5002                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
5003                         retry.error = e
5004                         continue
5005                 self._error_or_warning(e, fatal=fatal)
5006                 break
5007
5008             try:
5009                 self._extract_and_report_alerts(data)
5010             except ExtractorError as e:
5011                 self._error_or_warning(e, fatal=fatal)
5012                 break
5013
5014             # Sometimes youtube returns a webpage with incomplete ytInitialData
5015             # See: https://github.com/yt-dlp/yt-dlp/issues/116
5016             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
5017                 retry.error = ExtractorError('Incomplete yt initial data received')
5018                 continue
5019
5020         return webpage, data
5021
5022     def _report_playlist_authcheck(self, ytcfg, fatal=True):
5023         """Use if failed to extract ytcfg (and data) from initial webpage"""
5024         if not ytcfg and self.is_authenticated:
5025             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
5026             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
5027                 raise ExtractorError(
5028                     f'{msg}. If you are not downloading private content, or '
5029                     'your cookies are only for the first account and channel,'
5030                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
5031                     expected=True)
5032             self.report_warning(msg, only_once=True)
5033
5034     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
5035         data = None
5036         if not self.skip_webpage:
5037             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
5038             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
5039             # Reject webpage data if redirected to home page without explicitly requesting
5040             selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
5041             if (url != 'https://www.youtube.com/feed/recommended'
5042                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
5043                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
5044                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
5045                 if fatal:
5046                     raise ExtractorError(msg, expected=True)
5047                 self.report_warning(msg, only_once=True)
5048         if not data:
5049             self._report_playlist_authcheck(ytcfg, fatal=fatal)
5050             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
5051         return data, ytcfg
5052
5053     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
5054         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
5055         resolve_response = self._extract_response(
5056             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
5057             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
5058         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
5059         for ep_key, ep in endpoints.items():
5060             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
5061             if params:
5062                 return self._extract_response(
5063                     item_id=item_id, query=params, ep=ep, headers=headers,
5064                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
5065                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
5066         err_note = 'Failed to resolve url (does the playlist exist?)'
5067         if fatal:
5068             raise ExtractorError(err_note, expected=True)
5069         self.report_warning(err_note, item_id)
5070
5071     _SEARCH_PARAMS = None
5072
5073     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
5074         data = {'query': query}
5075         if params is NO_DEFAULT:
5076             params = self._SEARCH_PARAMS
5077         if params:
5078             data['params'] = params
5079
5080         content_keys = (
5081             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
5082             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
5083             # ytmusic search
5084             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
5085             ('continuationContents', ),
5086         )
5087         display_id = f'query "{query}"'
5088         check_get_keys = tuple({keys[0] for keys in content_keys})
5089         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
5090         self._report_playlist_authcheck(ytcfg, fatal=False)
5091
5092         continuation_list = [None]
5093         search = None
5094         for page_num in itertools.count(1):
5095             data.update(continuation_list[0] or {})
5096             headers = self.generate_api_headers(
5097                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
5098             search = self._extract_response(
5099                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
5100                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
5101             slr_contents = traverse_obj(search, *content_keys)
5102             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
5103             if not continuation_list[0]:
5104                 break
5105
5106
5107 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
5108     IE_DESC = 'YouTube Tabs'
5109     _VALID_URL = r'''(?x:
5110         https?://
5111             (?:\w+\.)?
5112             (?:
5113                 youtube(?:kids)?\.com|
5114                 %(invidious)s
5115             )/
5116             (?:
5117                 (?P<channel_type>channel|c|user|browse)/|
5118                 (?P<not_channel>
5119                     feed/|hashtag/|
5120                     (?:playlist|watch)\?.*?\blist=
5121                 )|
5122                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
5123             )
5124             (?P<id>[^/?\#&]+)
5125     )''' % {
5126         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
5127         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5128     }
5129     IE_NAME = 'youtube:tab'
5130
5131     _TESTS = [{
5132         'note': 'playlists, multipage',
5133         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
5134         'playlist_mincount': 94,
5135         'info_dict': {
5136             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5137             'title': 'Igor Kleiner - Playlists',
5138             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5139             'uploader': 'Igor Kleiner',
5140             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5141             'channel': 'Igor Kleiner',
5142             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5143             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5144             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5145             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5146             'channel_follower_count': int
5147         },
5148     }, {
5149         'note': 'playlists, multipage, different order',
5150         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5151         'playlist_mincount': 94,
5152         'info_dict': {
5153             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5154             'title': 'Igor Kleiner - Playlists',
5155             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5156             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5157             'uploader': 'Igor Kleiner',
5158             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5159             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5160             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5161             'channel': 'Igor Kleiner',
5162             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5163             'channel_follower_count': int
5164         },
5165     }, {
5166         'note': 'playlists, series',
5167         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5168         'playlist_mincount': 5,
5169         'info_dict': {
5170             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5171             'title': '3Blue1Brown - Playlists',
5172             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5173             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5174             'uploader': '3Blue1Brown',
5175             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5176             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5177             'channel': '3Blue1Brown',
5178             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5179             'tags': ['Mathematics'],
5180             'channel_follower_count': int
5181         },
5182     }, {
5183         'note': 'playlists, singlepage',
5184         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5185         'playlist_mincount': 4,
5186         'info_dict': {
5187             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5188             'title': 'ThirstForScience - Playlists',
5189             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5190             'uploader': 'ThirstForScience',
5191             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5192             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5193             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5194             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5195             'tags': 'count:13',
5196             'channel': 'ThirstForScience',
5197             'channel_follower_count': int
5198         }
5199     }, {
5200         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5201         'only_matching': True,
5202     }, {
5203         'note': 'basic, single video playlist',
5204         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5205         'info_dict': {
5206             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5207             'uploader': 'Sergey M.',
5208             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5209             'title': 'youtube-dl public playlist',
5210             'description': '',
5211             'tags': [],
5212             'view_count': int,
5213             'modified_date': '20201130',
5214             'channel': 'Sergey M.',
5215             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5216             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5217             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5218             'availability': 'public',
5219         },
5220         'playlist_count': 1,
5221     }, {
5222         'note': 'empty playlist',
5223         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5224         'info_dict': {
5225             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5226             'uploader': 'Sergey M.',
5227             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5228             'title': 'youtube-dl empty playlist',
5229             'tags': [],
5230             'channel': 'Sergey M.',
5231             'description': '',
5232             'modified_date': '20160902',
5233             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5234             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5235             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5236             'availability': 'public',
5237         },
5238         'playlist_count': 0,
5239     }, {
5240         'note': 'Home tab',
5241         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5242         'info_dict': {
5243             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5244             'title': 'lex will - Home',
5245             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5246             'uploader': 'lex will',
5247             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5248             'channel': 'lex will',
5249             'tags': ['bible', 'history', 'prophesy'],
5250             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5251             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5252             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5253             'channel_follower_count': int
5254         },
5255         'playlist_mincount': 2,
5256     }, {
5257         'note': 'Videos tab',
5258         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5259         'info_dict': {
5260             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5261             'title': 'lex will - Videos',
5262             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5263             'uploader': 'lex will',
5264             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5265             'tags': ['bible', 'history', 'prophesy'],
5266             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5267             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5268             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5269             'channel': 'lex will',
5270             'channel_follower_count': int
5271         },
5272         'playlist_mincount': 975,
5273     }, {
5274         'note': 'Videos tab, sorted by popular',
5275         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5276         'info_dict': {
5277             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5278             'title': 'lex will - Videos',
5279             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5280             'uploader': 'lex will',
5281             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5282             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5283             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5284             'channel': 'lex will',
5285             'tags': ['bible', 'history', 'prophesy'],
5286             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5287             'channel_follower_count': int
5288         },
5289         'playlist_mincount': 199,
5290     }, {
5291         'note': 'Playlists tab',
5292         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5293         'info_dict': {
5294             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5295             'title': 'lex will - Playlists',
5296             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5297             'uploader': 'lex will',
5298             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5299             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5300             'channel': 'lex will',
5301             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5302             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5303             'tags': ['bible', 'history', 'prophesy'],
5304             'channel_follower_count': int
5305         },
5306         'playlist_mincount': 17,
5307     }, {
5308         'note': 'Community tab',
5309         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5310         'info_dict': {
5311             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5312             'title': 'lex will - Community',
5313             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5314             'uploader': 'lex will',
5315             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5316             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5317             'channel': 'lex will',
5318             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5319             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5320             'tags': ['bible', 'history', 'prophesy'],
5321             'channel_follower_count': int
5322         },
5323         'playlist_mincount': 18,
5324     }, {
5325         'note': 'Channels tab',
5326         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5327         'info_dict': {
5328             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5329             'title': 'lex will - Channels',
5330             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5331             'uploader': 'lex will',
5332             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5333             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5334             'channel': 'lex will',
5335             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5336             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5337             'tags': ['bible', 'history', 'prophesy'],
5338             'channel_follower_count': int
5339         },
5340         'playlist_mincount': 12,
5341     }, {
5342         'note': 'Search tab',
5343         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5344         'playlist_mincount': 40,
5345         'info_dict': {
5346             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5347             'title': '3Blue1Brown - Search - linear algebra',
5348             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5349             'uploader': '3Blue1Brown',
5350             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5351             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5352             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5353             'tags': ['Mathematics'],
5354             'channel': '3Blue1Brown',
5355             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5356             'channel_follower_count': int
5357         },
5358     }, {
5359         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5360         'only_matching': True,
5361     }, {
5362         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5363         'only_matching': True,
5364     }, {
5365         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5366         'only_matching': True,
5367     }, {
5368         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5369         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5370         'info_dict': {
5371             'title': '29C3: Not my department',
5372             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5373             'uploader': 'Christiaan008',
5374             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5375             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5376             'tags': [],
5377             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5378             'view_count': int,
5379             'modified_date': '20150605',
5380             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5381             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5382             'channel': 'Christiaan008',
5383             'availability': 'public',
5384         },
5385         'playlist_count': 96,
5386     }, {
5387         'note': 'Large playlist',
5388         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5389         'info_dict': {
5390             'title': 'Uploads from Cauchemar',
5391             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5392             'uploader': 'Cauchemar',
5393             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5394             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5395             'tags': [],
5396             'modified_date': r're:\d{8}',
5397             'channel': 'Cauchemar',
5398             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5399             'view_count': int,
5400             'description': '',
5401             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5402             'availability': 'public',
5403         },
5404         'playlist_mincount': 1123,
5405         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5406     }, {
5407         'note': 'even larger playlist, 8832 videos',
5408         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5409         'only_matching': True,
5410     }, {
5411         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5412         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5413         'info_dict': {
5414             'title': 'Uploads from Interstellar Movie',
5415             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5416             'uploader': 'Interstellar Movie',
5417             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5418             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5419             'tags': [],
5420             'view_count': int,
5421             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5422             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5423             'channel': 'Interstellar Movie',
5424             'description': '',
5425             'modified_date': r're:\d{8}',
5426             'availability': 'public',
5427         },
5428         'playlist_mincount': 21,
5429     }, {
5430         'note': 'Playlist with "show unavailable videos" button',
5431         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5432         'info_dict': {
5433             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5434             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5435             'uploader': 'Phim Siêu Nhân Nhật Bản',
5436             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5437             'view_count': int,
5438             'channel': 'Phim Siêu Nhân Nhật Bản',
5439             'tags': [],
5440             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5441             'description': '',
5442             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5443             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5444             'modified_date': r're:\d{8}',
5445             'availability': 'public',
5446         },
5447         'playlist_mincount': 200,
5448         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5449     }, {
5450         'note': 'Playlist with unavailable videos in page 7',
5451         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5452         'info_dict': {
5453             'title': 'Uploads from BlankTV',
5454             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5455             'uploader': 'BlankTV',
5456             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5457             'channel': 'BlankTV',
5458             'channel_url': 'https://www.youtube.com/c/blanktv',
5459             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5460             'view_count': int,
5461             'tags': [],
5462             'uploader_url': 'https://www.youtube.com/c/blanktv',
5463             'modified_date': r're:\d{8}',
5464             'description': '',
5465             'availability': 'public',
5466         },
5467         'playlist_mincount': 1000,
5468         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5469     }, {
5470         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5471         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5472         'info_dict': {
5473             'title': 'Data Analysis with Dr Mike Pound',
5474             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5475             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5476             'uploader': 'Computerphile',
5477             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5478             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5479             'tags': [],
5480             'view_count': int,
5481             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5482             'channel_url': 'https://www.youtube.com/user/Computerphile',
5483             'channel': 'Computerphile',
5484             'availability': 'public',
5485             'modified_date': '20190712',
5486         },
5487         'playlist_mincount': 11,
5488     }, {
5489         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5490         'only_matching': True,
5491     }, {
5492         'note': 'Playlist URL that does not actually serve a playlist',
5493         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5494         'info_dict': {
5495             'id': 'FqZTN594JQw',
5496             'ext': 'webm',
5497             'title': "Smiley's People 01 detective, Adventure Series, Action",
5498             'uploader': 'STREEM',
5499             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5500             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5501             'upload_date': '20150526',
5502             'license': 'Standard YouTube License',
5503             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5504             'categories': ['People & Blogs'],
5505             'tags': list,
5506             'view_count': int,
5507             'like_count': int,
5508         },
5509         'params': {
5510             'skip_download': True,
5511         },
5512         'skip': 'This video is not available.',
5513         'add_ie': [YoutubeIE.ie_key()],
5514     }, {
5515         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5516         'only_matching': True,
5517     }, {
5518         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5519         'only_matching': True,
5520     }, {
5521         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5522         'info_dict': {
5523             'id': 'Wq15eF5vCbI',  # This will keep changing
5524             'ext': 'mp4',
5525             'title': str,
5526             'uploader': 'Sky News',
5527             'uploader_id': 'skynews',
5528             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5529             'upload_date': r're:\d{8}',
5530             'description': str,
5531             'categories': ['News & Politics'],
5532             'tags': list,
5533             'like_count': int,
5534             'release_timestamp': int,
5535             'channel': 'Sky News',
5536             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5537             'age_limit': 0,
5538             'view_count': int,
5539             'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
5540             'playable_in_embed': True,
5541             'release_date': r're:\d+',
5542             'availability': 'public',
5543             'live_status': 'is_live',
5544             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5545             'channel_follower_count': int,
5546             'concurrent_view_count': int,
5547         },
5548         'params': {
5549             'skip_download': True,
5550         },
5551         'expected_warnings': ['Ignoring subtitle tracks found in '],
5552     }, {
5553         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5554         'info_dict': {
5555             'id': 'a48o2S1cPoo',
5556             'ext': 'mp4',
5557             'title': 'The Young Turks - Live Main Show',
5558             'uploader': 'The Young Turks',
5559             'uploader_id': 'TheYoungTurks',
5560             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5561             'upload_date': '20150715',
5562             'license': 'Standard YouTube License',
5563             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5564             'categories': ['News & Politics'],
5565             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5566             'like_count': int,
5567         },
5568         'params': {
5569             'skip_download': True,
5570         },
5571         'only_matching': True,
5572     }, {
5573         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5574         'only_matching': True,
5575     }, {
5576         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5577         'only_matching': True,
5578     }, {
5579         'note': 'A channel that is not live. Should raise error',
5580         'url': 'https://www.youtube.com/user/numberphile/live',
5581         'only_matching': True,
5582     }, {
5583         'url': 'https://www.youtube.com/feed/trending',
5584         'only_matching': True,
5585     }, {
5586         'url': 'https://www.youtube.com/feed/library',
5587         'only_matching': True,
5588     }, {
5589         'url': 'https://www.youtube.com/feed/history',
5590         'only_matching': True,
5591     }, {
5592         'url': 'https://www.youtube.com/feed/subscriptions',
5593         'only_matching': True,
5594     }, {
5595         'url': 'https://www.youtube.com/feed/watch_later',
5596         'only_matching': True,
5597     }, {
5598         'note': 'Recommended - redirects to home page.',
5599         'url': 'https://www.youtube.com/feed/recommended',
5600         'only_matching': True,
5601     }, {
5602         'note': 'inline playlist with not always working continuations',
5603         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5604         'only_matching': True,
5605     }, {
5606         'url': 'https://www.youtube.com/course',
5607         'only_matching': True,
5608     }, {
5609         'url': 'https://www.youtube.com/zsecurity',
5610         'only_matching': True,
5611     }, {
5612         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5613         'only_matching': True,
5614     }, {
5615         'url': 'https://www.youtube.com/TheYoungTurks/live',
5616         'only_matching': True,
5617     }, {
5618         'url': 'https://www.youtube.com/hashtag/cctv9',
5619         'info_dict': {
5620             'id': 'cctv9',
5621             'title': '#cctv9',
5622             'tags': [],
5623         },
5624         'playlist_mincount': 300,  # not consistent but should be over 300
5625     }, {
5626         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5627         'only_matching': True,
5628     }, {
5629         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5630         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5631         'only_matching': True
5632     }, {
5633         'note': '/browse/ should redirect to /channel/',
5634         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5635         'only_matching': True
5636     }, {
5637         'note': 'VLPL, should redirect to playlist?list=PL...',
5638         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5639         'info_dict': {
5640             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5641             'uploader': 'NoCopyrightSounds',
5642             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5643             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5644             'title': 'NCS : All Releases 💿',
5645             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5646             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5647             'modified_date': r're:\d{8}',
5648             'view_count': int,
5649             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5650             'tags': [],
5651             'channel': 'NoCopyrightSounds',
5652             'availability': 'public',
5653         },
5654         'playlist_mincount': 166,
5655         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5656     }, {
5657         'note': 'Topic, should redirect to playlist?list=UU...',
5658         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5659         'info_dict': {
5660             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5661             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5662             'title': 'Uploads from Royalty Free Music - Topic',
5663             'uploader': 'Royalty Free Music - Topic',
5664             'tags': [],
5665             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5666             'channel': 'Royalty Free Music - Topic',
5667             'view_count': int,
5668             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5669             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5670             'modified_date': r're:\d{8}',
5671             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5672             'description': '',
5673             'availability': 'public',
5674         },
5675         'playlist_mincount': 101,
5676     }, {
5677         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
5678         # Treat as a general feed
5679         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5680         'info_dict': {
5681             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5682             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5683             'tags': [],
5684         },
5685         'playlist_mincount': 9,
5686     }, {
5687         'note': 'Youtube music Album',
5688         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5689         'info_dict': {
5690             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5691             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5692             'tags': [],
5693             'view_count': int,
5694             'description': '',
5695             'availability': 'unlisted',
5696             'modified_date': r're:\d{8}',
5697         },
5698         'playlist_count': 50,
5699     }, {
5700         'note': 'unlisted single video playlist',
5701         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5702         'info_dict': {
5703             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5704             'uploader': 'colethedj',
5705             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5706             'title': 'yt-dlp unlisted playlist test',
5707             'availability': 'unlisted',
5708             'tags': [],
5709             'modified_date': '20220418',
5710             'channel': 'colethedj',
5711             'view_count': int,
5712             'description': '',
5713             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5714             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5715             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5716         },
5717         'playlist_count': 1,
5718     }, {
5719         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5720         'url': 'https://www.youtube.com/feed/recommended',
5721         'info_dict': {
5722             'id': 'recommended',
5723             'title': 'recommended',
5724             'tags': [],
5725         },
5726         'playlist_mincount': 50,
5727         'params': {
5728             'skip_download': True,
5729             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5730         },
5731     }, {
5732         'note': 'API Fallback: /videos tab, sorted by oldest first',
5733         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5734         'info_dict': {
5735             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5736             'title': 'Cody\'sLab - Videos',
5737             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5738             'uploader': 'Cody\'sLab',
5739             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5740             'channel': 'Cody\'sLab',
5741             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5742             'tags': [],
5743             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5744             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5745             'channel_follower_count': int
5746         },
5747         'playlist_mincount': 650,
5748         'params': {
5749             'skip_download': True,
5750             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5751         },
5752         'skip': 'Query for sorting no longer works',
5753     }, {
5754         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5755         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5756         'info_dict': {
5757             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5758             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5759             'title': 'Uploads from Royalty Free Music - Topic',
5760             'uploader': 'Royalty Free Music - Topic',
5761             'modified_date': r're:\d{8}',
5762             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5763             'description': '',
5764             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5765             'tags': [],
5766             'channel': 'Royalty Free Music - Topic',
5767             'view_count': int,
5768             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5769             'availability': 'public',
5770         },
5771         'playlist_mincount': 101,
5772         'params': {
5773             'skip_download': True,
5774             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5775         },
5776     }, {
5777         'note': 'non-standard redirect to regional channel',
5778         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5779         'only_matching': True
5780     }, {
5781         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5782         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5783         'info_dict': {
5784             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5785             'modified_date': '20220407',
5786             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5787             'tags': [],
5788             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5789             'uploader': 'pukkandan',
5790             'availability': 'unlisted',
5791             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5792             'channel': 'pukkandan',
5793             'description': 'Test for collaborative playlist',
5794             'title': 'yt-dlp test - collaborative playlist',
5795             'view_count': int,
5796             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5797         },
5798         'playlist_mincount': 2
5799     }, {
5800         'note': 'translated tab name',
5801         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5802         'info_dict': {
5803             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5804             'tags': [],
5805             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5806             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5807             'description': 'test description',
5808             'title': 'cole-dlp-test-acc - 再生リスト',
5809             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5810             'uploader': 'cole-dlp-test-acc',
5811             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5812             'channel': 'cole-dlp-test-acc',
5813         },
5814         'playlist_mincount': 1,
5815         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5816         'expected_warnings': ['Preferring "ja"'],
5817     }, {
5818         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5819         'note': 'preferred lang set with playlist with translated video titles',
5820         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5821         'info_dict': {
5822             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5823             'tags': [],
5824             'view_count': int,
5825             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5826             'uploader': 'cole-dlp-test-acc',
5827             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5828             'channel': 'cole-dlp-test-acc',
5829             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5830             'description': 'test',
5831             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5832             'title': 'dlp test playlist',
5833             'availability': 'public',
5834         },
5835         'playlist_mincount': 1,
5836         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5837         'expected_warnings': ['Preferring "ja"'],
5838     }, {
5839         # shorts audio pivot for 2GtVksBMYFM.
5840         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5841         'info_dict': {
5842             'id': 'sfv_audio_pivot',
5843             'title': 'sfv_audio_pivot',
5844             'tags': [],
5845         },
5846         'playlist_mincount': 50,
5847
5848     }, {
5849         # Channel with a real live tab (not to be mistaken with streams tab)
5850         # Do not treat like it should redirect to live stream
5851         'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
5852         'info_dict': {
5853             'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
5854             'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
5855             'tags': [],
5856         },
5857         'playlist_mincount': 20,
5858     }, {
5859         # Tab name is not the same as tab id
5860         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
5861         'info_dict': {
5862             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5863             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
5864             'tags': [],
5865         },
5866         'playlist_mincount': 8,
5867     }, {
5868         # Home tab id is literally home. Not to get mistaken with featured
5869         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
5870         'info_dict': {
5871             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5872             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
5873             'tags': [],
5874         },
5875         'playlist_mincount': 8,
5876     }, {
5877         # Should get three playlists for videos, shorts and streams tabs
5878         'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5879         'info_dict': {
5880             'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5881             'title': 'Polka Ch. 尾丸ポルカ',
5882             'channel_follower_count': int,
5883             'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5884             'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5885             'uploader': 'Polka Ch. 尾丸ポルカ',
5886             'description': 'md5:3b8df1ac5af337aa206e37ee3d181ec9',
5887             'channel': 'Polka Ch. 尾丸ポルカ',
5888             'tags': 'count:35',
5889             'uploader_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5890             'uploader_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5891         },
5892         'playlist_count': 3,
5893     }, {
5894         # Shorts tab with channel with handle
5895         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
5896         'info_dict': {
5897             'id': 'UC0intLFzLaudFG-xAvUEO-A',
5898             'title': 'Not Just Bikes - Shorts',
5899             'tags': 'count:12',
5900             'uploader': 'Not Just Bikes',
5901             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5902             'description': 'md5:7513148b1f02b924783157d84c4ea555',
5903             'channel_follower_count': int,
5904             'uploader_id': 'UC0intLFzLaudFG-xAvUEO-A',
5905             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
5906             'uploader_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5907             'channel': 'Not Just Bikes',
5908         },
5909         'playlist_mincount': 10,
5910     }, {
5911         # Streams tab
5912         'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
5913         'info_dict': {
5914             'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5915             'title': '中村悠一 - Live',
5916             'tags': 'count:7',
5917             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5918             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5919             'uploader_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5920             'channel': '中村悠一',
5921             'uploader_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5922             'channel_follower_count': int,
5923             'uploader': '中村悠一',
5924             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
5925         },
5926         'playlist_mincount': 60,
5927     }, {
5928         # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
5929         # See test_youtube_lists
5930         'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
5931         'only_matching': True,
5932     }, {
5933         # No uploads and no UCID given. Should fail with no uploads error
5934         # See test_youtube_lists
5935         'url': 'https://www.youtube.com/news',
5936         'only_matching': True
5937     }, {
5938         # No videos tab but has a shorts tab
5939         'url': 'https://www.youtube.com/c/TKFShorts',
5940         'info_dict': {
5941             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5942             'title': 'Shorts Break - Shorts',
5943             'tags': 'count:32',
5944             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5945             'channel': 'Shorts Break',
5946             'description': 'md5:a6c234cf3d50d878ef8721e34457cd11',
5947             'uploader': 'Shorts Break',
5948             'channel_follower_count': int,
5949             'uploader_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
5950             'uploader_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
5951             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
5952         },
5953         'playlist_mincount': 30,
5954     }, {
5955         # Trending Now Tab. tab id is empty
5956         'url': 'https://www.youtube.com/feed/trending',
5957         'info_dict': {
5958             'id': 'trending',
5959             'title': 'trending - Now',
5960             'tags': [],
5961         },
5962         'playlist_mincount': 30,
5963     }, {
5964         # Trending Gaming Tab. tab id is empty
5965         'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
5966         'info_dict': {
5967             'id': 'trending',
5968             'title': 'trending - Gaming',
5969             'tags': [],
5970         },
5971         'playlist_mincount': 30,
5972     }, {
5973         # Shorts url result in shorts tab
5974         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
5975         'info_dict': {
5976             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5977             'title': 'cole-dlp-test-acc - Shorts',
5978             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5979             'channel': 'cole-dlp-test-acc',
5980             'description': 'test description',
5981             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5982             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5983             'tags': [],
5984             'uploader': 'cole-dlp-test-acc',
5985             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5986
5987         },
5988         'playlist': [{
5989             'info_dict': {
5990                 '_type': 'url',
5991                 'ie_key': 'Youtube',
5992                 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
5993                 'id': 'sSM9J5YH_60',
5994                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5995                 'title': 'SHORT short',
5996                 'channel': 'cole-dlp-test-acc',
5997                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5998                 'view_count': int,
5999                 'thumbnails': list,
6000             }
6001         }],
6002         'params': {'extract_flat': True},
6003     }, {
6004         # Live video status should be extracted
6005         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
6006         'info_dict': {
6007             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6008             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live',  # TODO, should be Minecraft - Live or Minecraft - Topic - Live
6009             'tags': []
6010         },
6011         'playlist': [{
6012             'info_dict': {
6013                 '_type': 'url',
6014                 'ie_key': 'Youtube',
6015                 'url': 'startswith:https://www.youtube.com/watch?v=',
6016                 'id': str,
6017                 'title': str,
6018                 'live_status': 'is_live',
6019                 'channel_id': str,
6020                 'channel_url': str,
6021                 'concurrent_view_count': int,
6022                 'channel': str,
6023             }
6024         }],
6025         'params': {'extract_flat': True, 'playlist_items': '1'},
6026         'playlist_mincount': 1
6027     }, {
6028         # Channel renderer metadata. Contains number of videos on the channel
6029         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
6030         'info_dict': {
6031             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6032             'title': 'cole-dlp-test-acc - Channels',
6033             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
6034             'channel': 'cole-dlp-test-acc',
6035             'description': 'test description',
6036             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6037             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6038             'tags': [],
6039             'uploader': 'cole-dlp-test-acc',
6040             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6041
6042         },
6043         'playlist': [{
6044             'info_dict': {
6045                 '_type': 'url',
6046                 'ie_key': 'YoutubeTab',
6047                 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6048                 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6049                 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6050                 'title': 'PewDiePie',
6051                 'channel': 'PewDiePie',
6052                 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6053                 'thumbnails': list,
6054                 'channel_follower_count': int,
6055                 'playlist_count': int
6056             }
6057         }],
6058         'params': {'extract_flat': True},
6059     }]
6060
6061     @classmethod
6062     def suitable(cls, url):
6063         return False if YoutubeIE.suitable(url) else super().suitable(url)
6064
6065     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
6066
6067     def _get_url_mobj(self, url):
6068         mobj = self._URL_RE.match(url).groupdict()
6069         mobj.update((k, '') for k, v in mobj.items() if v is None)
6070         return mobj
6071
6072     def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
6073         tab_name = (tab.get('title') or '').lower()
6074         tab_url = urljoin(base_url, traverse_obj(
6075             tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
6076
6077         tab_id = (tab_url and self._get_url_mobj(tab_url)['tab'][1:]
6078                   or traverse_obj(tab, 'tabIdentifier', expected_type=str))
6079         if tab_id:
6080             return {
6081                 'TAB_ID_SPONSORSHIPS': 'membership',
6082             }.get(tab_id, tab_id), tab_name
6083
6084         # Fallback to tab name if we cannot get the tab id.
6085         # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
6086         # Note that in the case of translated tab name this may result in an empty string, which we don't want.
6087         if tab_name:
6088             self.write_debug(f'Falling back to selected tab name: {tab_name}')
6089         return {
6090             'home': 'featured',
6091             'live': 'streams',
6092         }.get(tab_name, tab_name), tab_name
6093
6094     def _has_tab(self, tabs, tab_id):
6095         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
6096
6097     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
6098     def _real_extract(self, url, smuggled_data):
6099         item_id = self._match_id(url)
6100         url = urllib.parse.urlunparse(
6101             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
6102         compat_opts = self.get_param('compat_opts', [])
6103
6104         mobj = self._get_url_mobj(url)
6105         pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
6106         if is_channel and smuggled_data.get('is_music_url'):
6107             if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
6108                 return self.url_result(
6109                     f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
6110             elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
6111                 mdata = self._extract_tab_endpoint(
6112                     f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
6113                 murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
6114                                     get_all=False, expected_type=str)
6115                 if not murl:
6116                     raise ExtractorError('Failed to resolve album to playlist')
6117                 return self.url_result(murl, YoutubeTabIE)
6118             elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
6119                 return self.url_result(
6120                     f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
6121
6122         original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
6123         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
6124             url = f'{pre}/videos{post}'
6125
6126         # Handle both video/playlist URLs
6127         qs = parse_qs(url)
6128         video_id, playlist_id = [traverse_obj(qs, (key, 0)) for key in ('v', 'list')]
6129         if not video_id and mobj['not_channel'].startswith('watch'):
6130             if not playlist_id:
6131                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
6132                 raise ExtractorError('A video URL was given without video ID', expected=True)
6133             # Common mistake: https://www.youtube.com/watch?list=playlist_id
6134             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
6135             return self.url_result(
6136                 f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
6137
6138         if not self._yes_playlist(playlist_id, video_id):
6139             return self.url_result(
6140                 f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6141
6142         data, ytcfg = self._extract_data(url, display_id)
6143
6144         # YouTube may provide a non-standard redirect to the regional channel
6145         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
6146         # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
6147         redirect_url = traverse_obj(
6148             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
6149         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
6150             redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
6151             self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
6152             return self.url_result(redirect_url, YoutubeTabIE)
6153
6154         tabs, extra_tabs = self._extract_tab_renderers(data), []
6155         if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
6156             selected_tab = self._extract_selected_tab(tabs)
6157             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
6158             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
6159
6160             if not original_tab_id and selected_tab_name:
6161                 self.to_screen('Downloading all uploads of the channel. '
6162                                'To download only the videos in a specific tab, pass the tab\'s URL')
6163                 if self._has_tab(tabs, 'streams'):
6164                     extra_tabs.append(''.join((pre, '/streams', post)))
6165                 if self._has_tab(tabs, 'shorts'):
6166                     extra_tabs.append(''.join((pre, '/shorts', post)))
6167                 # XXX: Members-only tab should also be extracted
6168
6169                 if not extra_tabs and selected_tab_id != 'videos':
6170                     # Channel does not have streams, shorts or videos tabs
6171                     if item_id[:2] != 'UC':
6172                         raise ExtractorError('This channel has no uploads', expected=True)
6173
6174                     # Topic channels don't have /videos. Use the equivalent playlist instead
6175                     pl_id = f'UU{item_id[2:]}'
6176                     pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
6177                     try:
6178                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
6179                     except ExtractorError:
6180                         raise ExtractorError('This channel has no uploads', expected=True)
6181                     else:
6182                         item_id, url = pl_id, pl_url
6183                         self.to_screen(
6184                             f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
6185
6186                 elif extra_tabs and selected_tab_id != 'videos':
6187                     # When there are shorts/live tabs but not videos tab
6188                     url, data = f'{pre}{post}', None
6189
6190             elif (original_tab_id or 'videos') != selected_tab_id:
6191                 if original_tab_id == 'live':
6192                     # Live tab should have redirected to the video
6193                     # Except in the case the channel has an actual live tab
6194                     # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
6195                     raise UserNotLive(video_id=item_id)
6196                 elif selected_tab_name:
6197                     raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
6198
6199                 # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
6200                 url = f'{pre}{post}'
6201
6202         # YouTube sometimes provides a button to reload playlist with unavailable videos.
6203         if 'no-youtube-unavailable-videos' not in compat_opts:
6204             data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
6205         self._extract_and_report_alerts(data, only_once=True)
6206
6207         tabs, entries = self._extract_tab_renderers(data), []
6208         if tabs:
6209             entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
6210             entries[0].update({
6211                 'extractor_key': YoutubeTabIE.ie_key(),
6212                 'extractor': YoutubeTabIE.IE_NAME,
6213                 'webpage_url': url,
6214             })
6215         if self.get_param('playlist_items') == '0':
6216             entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
6217         else:  # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
6218             entries.extend(map(self._real_extract, extra_tabs))
6219
6220         if len(entries) == 1:
6221             return entries[0]
6222         elif entries:
6223             metadata = self._extract_metadata_from_tabs(item_id, data)
6224             uploads_url = 'the Uploads (UU) playlist URL'
6225             if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
6226                 uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
6227             self.to_screen(
6228                 'Downloading as multiple playlists, separated by tabs. '
6229                 f'To download as a single playlist instead, pass {uploads_url}')
6230             return self.playlist_result(entries, item_id, **metadata)
6231
6232         # Inline playlist
6233         playlist = traverse_obj(
6234             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
6235         if playlist:
6236             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
6237
6238         video_id = traverse_obj(
6239             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
6240         if video_id:
6241             if tab != '/live':  # live tab is expected to redirect to video
6242                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
6243             return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6244
6245         raise ExtractorError('Unable to recognize tab page')
6246
6247
6248 class YoutubePlaylistIE(InfoExtractor):
6249     IE_DESC = 'YouTube playlists'
6250     _VALID_URL = r'''(?x)(?:
6251                         (?:https?://)?
6252                         (?:\w+\.)?
6253                         (?:
6254                             (?:
6255                                 youtube(?:kids)?\.com|
6256                                 %(invidious)s
6257                             )
6258                             /.*?\?.*?\blist=
6259                         )?
6260                         (?P<id>%(playlist_id)s)
6261                      )''' % {
6262         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
6263         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
6264     }
6265     IE_NAME = 'youtube:playlist'
6266     _TESTS = [{
6267         'note': 'issue #673',
6268         'url': 'PLBB231211A4F62143',
6269         'info_dict': {
6270             'title': '[OLD]Team Fortress 2 (Class-based LP)',
6271             'id': 'PLBB231211A4F62143',
6272             'uploader': 'Wickman',
6273             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6274             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
6275             'view_count': int,
6276             'uploader_url': 'https://www.youtube.com/c/WickmanVT',
6277             'modified_date': r're:\d{8}',
6278             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6279             'channel': 'Wickman',
6280             'tags': [],
6281             'channel_url': 'https://www.youtube.com/c/WickmanVT',
6282             'availability': 'public',
6283         },
6284         'playlist_mincount': 29,
6285     }, {
6286         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6287         'info_dict': {
6288             'title': 'YDL_safe_search',
6289             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6290         },
6291         'playlist_count': 2,
6292         'skip': 'This playlist is private',
6293     }, {
6294         'note': 'embedded',
6295         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6296         'playlist_count': 4,
6297         'info_dict': {
6298             'title': 'JODA15',
6299             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6300             'uploader': 'milan',
6301             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6302             'description': '',
6303             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6304             'tags': [],
6305             'modified_date': '20140919',
6306             'view_count': int,
6307             'channel': 'milan',
6308             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6309             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6310             'availability': 'public',
6311         },
6312         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
6313     }, {
6314         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6315         'playlist_mincount': 455,
6316         'info_dict': {
6317             'title': '2018 Chinese New Singles (11/6 updated)',
6318             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6319             'uploader': 'LBK',
6320             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6321             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
6322             'channel': 'LBK',
6323             'view_count': int,
6324             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
6325             'tags': [],
6326             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
6327             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6328             'modified_date': r're:\d{8}',
6329             'availability': 'public',
6330         },
6331         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6332     }, {
6333         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
6334         'only_matching': True,
6335     }, {
6336         # music album playlist
6337         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
6338         'only_matching': True,
6339     }]
6340
6341     @classmethod
6342     def suitable(cls, url):
6343         if YoutubeTabIE.suitable(url):
6344             return False
6345         from ..utils import parse_qs
6346         qs = parse_qs(url)
6347         if qs.get('v', [None])[0]:
6348             return False
6349         return super().suitable(url)
6350
6351     def _real_extract(self, url):
6352         playlist_id = self._match_id(url)
6353         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
6354         url = update_url_query(
6355             'https://www.youtube.com/playlist',
6356             parse_qs(url) or {'list': playlist_id})
6357         if is_music_url:
6358             url = smuggle_url(url, {'is_music_url': True})
6359         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6360
6361
6362 class YoutubeYtBeIE(InfoExtractor):
6363     IE_DESC = 'youtu.be'
6364     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
6365     _TESTS = [{
6366         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
6367         'info_dict': {
6368             'id': 'yeWKywCrFtk',
6369             'ext': 'mp4',
6370             'title': 'Small Scale Baler and Braiding Rugs',
6371             'uploader': 'Backus-Page House Museum',
6372             'uploader_id': 'backuspagemuseum',
6373             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
6374             'upload_date': '20161008',
6375             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
6376             'categories': ['Nonprofits & Activism'],
6377             'tags': list,
6378             'like_count': int,
6379             'age_limit': 0,
6380             'playable_in_embed': True,
6381             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
6382             'channel': 'Backus-Page House Museum',
6383             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
6384             'live_status': 'not_live',
6385             'view_count': int,
6386             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
6387             'availability': 'public',
6388             'duration': 59,
6389             'comment_count': int,
6390             'channel_follower_count': int
6391         },
6392         'params': {
6393             'noplaylist': True,
6394             'skip_download': True,
6395         },
6396     }, {
6397         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6398         'only_matching': True,
6399     }]
6400
6401     def _real_extract(self, url):
6402         mobj = self._match_valid_url(url)
6403         video_id = mobj.group('id')
6404         playlist_id = mobj.group('playlist_id')
6405         return self.url_result(
6406             update_url_query('https://www.youtube.com/watch', {
6407                 'v': video_id,
6408                 'list': playlist_id,
6409                 'feature': 'youtu.be',
6410             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6411
6412
6413 class YoutubeLivestreamEmbedIE(InfoExtractor):
6414     IE_DESC = 'YouTube livestream embeds'
6415     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6416     _TESTS = [{
6417         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6418         'only_matching': True,
6419     }]
6420
6421     def _real_extract(self, url):
6422         channel_id = self._match_id(url)
6423         return self.url_result(
6424             f'https://www.youtube.com/channel/{channel_id}/live',
6425             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6426
6427
6428 class YoutubeYtUserIE(InfoExtractor):
6429     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6430     IE_NAME = 'youtube:user'
6431     _VALID_URL = r'ytuser:(?P<id>.+)'
6432     _TESTS = [{
6433         'url': 'ytuser:phihag',
6434         'only_matching': True,
6435     }]
6436
6437     def _real_extract(self, url):
6438         user_id = self._match_id(url)
6439         return self.url_result(f'https://www.youtube.com/user/{user_id}', YoutubeTabIE, user_id)
6440
6441
6442 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6443     IE_NAME = 'youtube:favorites'
6444     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6445     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6446     _LOGIN_REQUIRED = True
6447     _TESTS = [{
6448         'url': ':ytfav',
6449         'only_matching': True,
6450     }, {
6451         'url': ':ytfavorites',
6452         'only_matching': True,
6453     }]
6454
6455     def _real_extract(self, url):
6456         return self.url_result(
6457             'https://www.youtube.com/playlist?list=LL',
6458             ie=YoutubeTabIE.ie_key())
6459
6460
6461 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6462     IE_NAME = 'youtube:notif'
6463     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6464     _VALID_URL = r':ytnotif(?:ication)?s?'
6465     _LOGIN_REQUIRED = True
6466     _TESTS = [{
6467         'url': ':ytnotif',
6468         'only_matching': True,
6469     }, {
6470         'url': ':ytnotifications',
6471         'only_matching': True,
6472     }]
6473
6474     def _extract_notification_menu(self, response, continuation_list):
6475         notification_list = traverse_obj(
6476             response,
6477             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6478             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6479             expected_type=list) or []
6480         continuation_list[0] = None
6481         for item in notification_list:
6482             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6483             if entry:
6484                 yield entry
6485             continuation = item.get('continuationItemRenderer')
6486             if continuation:
6487                 continuation_list[0] = continuation
6488
6489     def _extract_notification_renderer(self, notification):
6490         video_id = traverse_obj(
6491             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6492         url = f'https://www.youtube.com/watch?v={video_id}'
6493         channel_id = None
6494         if not video_id:
6495             browse_ep = traverse_obj(
6496                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6497             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6498             post_id = self._search_regex(
6499                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6500                 'post id', default=None)
6501             if not channel_id or not post_id:
6502                 return
6503             # The direct /post url redirects to this in the browser
6504             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6505
6506         channel = traverse_obj(
6507             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6508             expected_type=str)
6509         notification_title = self._get_text(notification, 'shortMessage')
6510         if notification_title:
6511             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6512         # TODO: handle recommended videos
6513         title = self._search_regex(
6514             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6515             'video title', default=None)
6516         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6517                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6518                      else None)
6519         return {
6520             '_type': 'url',
6521             'url': url,
6522             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6523             'video_id': video_id,
6524             'title': title,
6525             'channel_id': channel_id,
6526             'channel': channel,
6527             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6528             'timestamp': timestamp,
6529         }
6530
6531     def _notification_menu_entries(self, ytcfg):
6532         continuation_list = [None]
6533         response = None
6534         for page in itertools.count(1):
6535             ctoken = traverse_obj(
6536                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6537             response = self._extract_response(
6538                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6539                 ep='notification/get_notification_menu', check_get_keys='actions',
6540                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6541             yield from self._extract_notification_menu(response, continuation_list)
6542             if not continuation_list[0]:
6543                 break
6544
6545     def _real_extract(self, url):
6546         display_id = 'notifications'
6547         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6548         self._report_playlist_authcheck(ytcfg)
6549         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6550
6551
6552 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6553     IE_DESC = 'YouTube search'
6554     IE_NAME = 'youtube:search'
6555     _SEARCH_KEY = 'ytsearch'
6556     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6557     _TESTS = [{
6558         'url': 'ytsearch5:youtube-dl test video',
6559         'playlist_count': 5,
6560         'info_dict': {
6561             'id': 'youtube-dl test video',
6562             'title': 'youtube-dl test video',
6563         }
6564     }]
6565
6566
6567 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6568     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6569     _SEARCH_KEY = 'ytsearchdate'
6570     IE_DESC = 'YouTube search, newest videos first'
6571     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6572     _TESTS = [{
6573         'url': 'ytsearchdate5:youtube-dl test video',
6574         'playlist_count': 5,
6575         'info_dict': {
6576             'id': 'youtube-dl test video',
6577             'title': 'youtube-dl test video',
6578         }
6579     }]
6580
6581
6582 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6583     IE_DESC = 'YouTube search URLs with sorting and filter support'
6584     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6585     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6586     _TESTS = [{
6587         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6588         'playlist_mincount': 5,
6589         'info_dict': {
6590             'id': 'youtube-dl test video',
6591             'title': 'youtube-dl test video',
6592         }
6593     }, {
6594         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6595         'playlist_mincount': 5,
6596         'info_dict': {
6597             'id': 'python',
6598             'title': 'python',
6599         }
6600     }, {
6601         'url': 'https://www.youtube.com/results?search_query=%23cats',
6602         'playlist_mincount': 1,
6603         'info_dict': {
6604             'id': '#cats',
6605             'title': '#cats',
6606             # The test suite does not have support for nested playlists
6607             # 'entries': [{
6608             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6609             #     'title': '#cats',
6610             # }],
6611         },
6612     }, {
6613         # Channel results
6614         'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
6615         'info_dict': {
6616             'id': 'kurzgesagt',
6617             'title': 'kurzgesagt',
6618         },
6619         'playlist': [{
6620             'info_dict': {
6621                 '_type': 'url',
6622                 'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6623                 'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6624                 'ie_key': 'YoutubeTab',
6625                 'channel': 'Kurzgesagt – In a Nutshell',
6626                 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
6627                 'title': 'Kurzgesagt – In a Nutshell',
6628                 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6629                 'playlist_count': int,  # XXX: should have a way of saying > 1
6630                 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6631                 'thumbnails': list
6632             }
6633         }],
6634         'params': {'extract_flat': True, 'playlist_items': '1'},
6635         'playlist_mincount': 1,
6636     }, {
6637         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6638         'only_matching': True,
6639     }]
6640
6641     def _real_extract(self, url):
6642         qs = parse_qs(url)
6643         query = (qs.get('search_query') or qs.get('q'))[0]
6644         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6645
6646
6647 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6648     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6649     IE_NAME = 'youtube:music:search_url'
6650     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6651     _TESTS = [{
6652         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6653         'playlist_count': 16,
6654         'info_dict': {
6655             'id': 'royalty free music',
6656             'title': 'royalty free music',
6657         }
6658     }, {
6659         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6660         'playlist_mincount': 30,
6661         'info_dict': {
6662             'id': 'royalty free music - songs',
6663             'title': 'royalty free music - songs',
6664         },
6665         'params': {'extract_flat': 'in_playlist'}
6666     }, {
6667         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6668         'playlist_mincount': 30,
6669         'info_dict': {
6670             'id': 'royalty free music - community playlists',
6671             'title': 'royalty free music - community playlists',
6672         },
6673         'params': {'extract_flat': 'in_playlist'}
6674     }]
6675
6676     _SECTIONS = {
6677         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6678         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6679         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6680         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6681         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6682         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6683     }
6684
6685     def _real_extract(self, url):
6686         qs = parse_qs(url)
6687         query = (qs.get('search_query') or qs.get('q'))[0]
6688         params = qs.get('sp', (None,))[0]
6689         if params:
6690             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6691         else:
6692             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6693             params = self._SECTIONS.get(section)
6694             if not params:
6695                 section = None
6696         title = join_nonempty(query, section, delim=' - ')
6697         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6698
6699
6700 class YoutubeFeedsInfoExtractor(InfoExtractor):
6701     """
6702     Base class for feed extractors
6703     Subclasses must re-define the _FEED_NAME property.
6704     """
6705     _LOGIN_REQUIRED = True
6706     _FEED_NAME = 'feeds'
6707
6708     def _real_initialize(self):
6709         YoutubeBaseInfoExtractor._check_login_required(self)
6710
6711     @classproperty
6712     def IE_NAME(self):
6713         return f'youtube:{self._FEED_NAME}'
6714
6715     def _real_extract(self, url):
6716         return self.url_result(
6717             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6718
6719
6720 class YoutubeWatchLaterIE(InfoExtractor):
6721     IE_NAME = 'youtube:watchlater'
6722     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6723     _VALID_URL = r':ytwatchlater'
6724     _TESTS = [{
6725         'url': ':ytwatchlater',
6726         'only_matching': True,
6727     }]
6728
6729     def _real_extract(self, url):
6730         return self.url_result(
6731             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6732
6733
6734 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6735     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6736     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6737     _FEED_NAME = 'recommended'
6738     _LOGIN_REQUIRED = False
6739     _TESTS = [{
6740         'url': ':ytrec',
6741         'only_matching': True,
6742     }, {
6743         'url': ':ytrecommended',
6744         'only_matching': True,
6745     }, {
6746         'url': 'https://youtube.com',
6747         'only_matching': True,
6748     }]
6749
6750
6751 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6752     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6753     _VALID_URL = r':ytsub(?:scription)?s?'
6754     _FEED_NAME = 'subscriptions'
6755     _TESTS = [{
6756         'url': ':ytsubs',
6757         'only_matching': True,
6758     }, {
6759         'url': ':ytsubscriptions',
6760         'only_matching': True,
6761     }]
6762
6763
6764 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6765     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6766     _VALID_URL = r':ythis(?:tory)?'
6767     _FEED_NAME = 'history'
6768     _TESTS = [{
6769         'url': ':ythistory',
6770         'only_matching': True,
6771     }]
6772
6773
6774 class YoutubeStoriesIE(InfoExtractor):
6775     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6776     IE_NAME = 'youtube:stories'
6777     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6778     _TESTS = [{
6779         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6780         'only_matching': True,
6781     }]
6782
6783     def _real_extract(self, url):
6784         playlist_id = f'RLTD{self._match_id(url)}'
6785         return self.url_result(
6786             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6787             ie=YoutubeTabIE, video_id=playlist_id)
6788
6789
6790 class YoutubeShortsAudioPivotIE(InfoExtractor):
6791     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6792     IE_NAME = 'youtube:shorts:pivot:audio'
6793     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6794     _TESTS = [{
6795         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6796         'only_matching': True,
6797     }]
6798
6799     @staticmethod
6800     def _generate_audio_pivot_params(video_id):
6801         """
6802         Generates sfv_audio_pivot browse params for this video id
6803         """
6804         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6805         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6806
6807     def _real_extract(self, url):
6808         video_id = self._match_id(url)
6809         return self.url_result(
6810             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6811             ie=YoutubeTabIE)
6812
6813
6814 class YoutubeTruncatedURLIE(InfoExtractor):
6815     IE_NAME = 'youtube:truncated_url'
6816     IE_DESC = False  # Do not list
6817     _VALID_URL = r'''(?x)
6818         (?:https?://)?
6819         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6820         (?:watch\?(?:
6821             feature=[a-z_]+|
6822             annotation_id=annotation_[^&]+|
6823             x-yt-cl=[0-9]+|
6824             hl=[^&]*|
6825             t=[0-9]+
6826         )?
6827         |
6828             attribution_link\?a=[^&]+
6829         )
6830         $
6831     '''
6832
6833     _TESTS = [{
6834         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6835         'only_matching': True,
6836     }, {
6837         'url': 'https://www.youtube.com/watch?',
6838         'only_matching': True,
6839     }, {
6840         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6841         'only_matching': True,
6842     }, {
6843         'url': 'https://www.youtube.com/watch?feature=foo',
6844         'only_matching': True,
6845     }, {
6846         'url': 'https://www.youtube.com/watch?hl=en-GB',
6847         'only_matching': True,
6848     }, {
6849         'url': 'https://www.youtube.com/watch?t=2372',
6850         'only_matching': True,
6851     }]
6852
6853     def _real_extract(self, url):
6854         raise ExtractorError(
6855             'Did you forget to quote the URL? Remember that & is a meta '
6856             'character in most shells, so you want to put the URL in quotes, '
6857             'like  youtube-dl '
6858             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6859             ' or simply  youtube-dl BaW_jenozKc  .',
6860             expected=True)
6861
6862
6863 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6864     IE_NAME = 'youtube:clip'
6865     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6866     _TESTS = [{
6867         # FIXME: Other metadata should be extracted from the clip, not from the base video
6868         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6869         'info_dict': {
6870             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6871             'ext': 'mp4',
6872             'section_start': 29.0,
6873             'section_end': 39.7,
6874             'duration': 10.7,
6875             'age_limit': 0,
6876             'availability': 'public',
6877             'categories': ['Gaming'],
6878             'channel': 'Scott The Woz',
6879             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6880             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6881             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6882             'like_count': int,
6883             'playable_in_embed': True,
6884             'tags': 'count:17',
6885             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6886             'title': 'Mobile Games on Console - Scott The Woz',
6887             'upload_date': '20210920',
6888             'uploader': 'Scott The Woz',
6889             'uploader_id': 'scottthewoz',
6890             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6891             'view_count': int,
6892             'live_status': 'not_live',
6893             'channel_follower_count': int
6894         }
6895     }]
6896
6897     def _real_extract(self, url):
6898         clip_id = self._match_id(url)
6899         _, data = self._extract_webpage(url, clip_id)
6900
6901         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6902         if not video_id:
6903             raise ExtractorError('Unable to find video ID')
6904
6905         clip_data = traverse_obj(data, (
6906             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6907             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6908             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6909             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6910
6911         return {
6912             '_type': 'url_transparent',
6913             'url': f'https://www.youtube.com/watch?v={video_id}',
6914             'ie_key': YoutubeIE.ie_key(),
6915             'id': clip_id,
6916             'section_start': int(clip_data['startTimeMs']) / 1000,
6917             'section_end': int(clip_data['endTimeMs']) / 1000,
6918         }
6919
6920
6921 class YoutubeTruncatedIDIE(InfoExtractor):
6922     IE_NAME = 'youtube:truncated_id'
6923     IE_DESC = False  # Do not list
6924     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6925
6926     _TESTS = [{
6927         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6928         'only_matching': True,
6929     }]
6930
6931     def _real_extract(self, url):
6932         video_id = self._match_id(url)
6933         raise ExtractorError(
6934             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6935             expected=True)