yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import enum
   6 import hashlib
   7 import itertools
   8 import json
   9 import math
  10 import os.path
  11 import random
  12 import re
  13 import sys
  14 import threading
  15 import time
  16 import traceback
  17 import urllib.error
  18 import urllib.parse
  19
  20 from .common import InfoExtractor, SearchInfoExtractor
  21 from .openload import PhantomJSwrapper
  22 from ..compat import functools
  23 from ..jsinterp import JSInterpreter
  24 from ..utils import (
  25     NO_DEFAULT,
  26     ExtractorError,
  27     LazyList,
  28     UserNotLive,
  29     bug_reports_message,
  30     classproperty,
  31     clean_html,
  32     datetime_from_str,
  33     dict_get,
  34     filter_dict,
  35     float_or_none,
  36     format_field,
  37     get_first,
  38     int_or_none,
  39     is_html,
  40     join_nonempty,
  41     js_to_json,
  42     mimetype2ext,
  43     network_exceptions,
  44     orderedSet,
  45     parse_codecs,
  46     parse_count,
  47     parse_duration,
  48     parse_iso8601,
  49     parse_qs,
  50     qualities,
  51     remove_start,
  52     smuggle_url,
  53     str_or_none,
  54     str_to_int,
  55     strftime_or_none,
  56     traverse_obj,
  57     try_get,
  58     unescapeHTML,
  59     unified_strdate,
  60     unified_timestamp,
  61     unsmuggle_url,
  62     update_url_query,
  63     url_or_none,
  64     urljoin,
  65     variadic,
  66 )
  67
  68 # any clients starting with _ cannot be explicitly requested by the user
  69 INNERTUBE_CLIENTS = {
  70     'web': {
  71         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  72         'INNERTUBE_CONTEXT': {
  73             'client': {
  74                 'clientName': 'WEB',
  75                 'clientVersion': '2.20220801.00.00',
  76             }
  77         },
  78         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  79     },
  80     'web_embedded': {
  81         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  82         'INNERTUBE_CONTEXT': {
  83             'client': {
  84                 'clientName': 'WEB_EMBEDDED_PLAYER',
  85                 'clientVersion': '1.20220731.00.00',
  86             },
  87         },
  88         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  89     },
  90     'web_music': {
  91         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  92         'INNERTUBE_HOST': 'music.youtube.com',
  93         'INNERTUBE_CONTEXT': {
  94             'client': {
  95                 'clientName': 'WEB_REMIX',
  96                 'clientVersion': '1.20220727.01.00',
  97             }
  98         },
  99         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 100     },
 101     'web_creator': {
 102         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 103         'INNERTUBE_CONTEXT': {
 104             'client': {
 105                 'clientName': 'WEB_CREATOR',
 106                 'clientVersion': '1.20220726.00.00',
 107             }
 108         },
 109         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 110     },
 111     'android': {
 112         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 113         'INNERTUBE_CONTEXT': {
 114             'client': {
 115                 'clientName': 'ANDROID',
 116                 'clientVersion': '17.31.35',
 117                 'androidSdkVersion': 30,
 118                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 119             }
 120         },
 121         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 122         'REQUIRE_JS_PLAYER': False
 123     },
 124     'android_embedded': {
 125         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 126         'INNERTUBE_CONTEXT': {
 127             'client': {
 128                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 129                 'clientVersion': '17.31.35',
 130                 'androidSdkVersion': 30,
 131                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 132             },
 133         },
 134         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 135         'REQUIRE_JS_PLAYER': False
 136     },
 137     'android_music': {
 138         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 139         'INNERTUBE_CONTEXT': {
 140             'client': {
 141                 'clientName': 'ANDROID_MUSIC',
 142                 'clientVersion': '5.16.51',
 143                 'androidSdkVersion': 30,
 144                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 145             }
 146         },
 147         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 148         'REQUIRE_JS_PLAYER': False
 149     },
 150     'android_creator': {
 151         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 152         'INNERTUBE_CONTEXT': {
 153             'client': {
 154                 'clientName': 'ANDROID_CREATOR',
 155                 'clientVersion': '22.30.100',
 156                 'androidSdkVersion': 30,
 157                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 158             },
 159         },
 160         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 161         'REQUIRE_JS_PLAYER': False
 162     },
 163     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 164     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 165     'ios': {
 166         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 167         'INNERTUBE_CONTEXT': {
 168             'client': {
 169                 'clientName': 'IOS',
 170                 'clientVersion': '17.33.2',
 171                 'deviceModel': 'iPhone14,3',
 172                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 173             }
 174         },
 175         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 176         'REQUIRE_JS_PLAYER': False
 177     },
 178     'ios_embedded': {
 179         'INNERTUBE_CONTEXT': {
 180             'client': {
 181                 'clientName': 'IOS_MESSAGES_EXTENSION',
 182                 'clientVersion': '17.33.2',
 183                 'deviceModel': 'iPhone14,3',
 184                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 185             },
 186         },
 187         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 188         'REQUIRE_JS_PLAYER': False
 189     },
 190     'ios_music': {
 191         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 192         'INNERTUBE_CONTEXT': {
 193             'client': {
 194                 'clientName': 'IOS_MUSIC',
 195                 'clientVersion': '5.21',
 196                 'deviceModel': 'iPhone14,3',
 197                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 198             },
 199         },
 200         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 201         'REQUIRE_JS_PLAYER': False
 202     },
 203     'ios_creator': {
 204         'INNERTUBE_CONTEXT': {
 205             'client': {
 206                 'clientName': 'IOS_CREATOR',
 207                 'clientVersion': '22.33.101',
 208                 'deviceModel': 'iPhone14,3',
 209                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 210             },
 211         },
 212         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 213         'REQUIRE_JS_PLAYER': False
 214     },
 215     # mweb has 'ultralow' formats
 216     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 217     'mweb': {
 218         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 219         'INNERTUBE_CONTEXT': {
 220             'client': {
 221                 'clientName': 'MWEB',
 222                 'clientVersion': '2.20220801.00.00',
 223             }
 224         },
 225         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 226     },
 227     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 228     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 229     'tv_embedded': {
 230         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 231         'INNERTUBE_CONTEXT': {
 232             'client': {
 233                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 234                 'clientVersion': '2.0',
 235             },
 236         },
 237         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 238     },
 239 }
 240
 241
 242 def _split_innertube_client(client_name):
 243     variant, *base = client_name.rsplit('.', 1)
 244     if base:
 245         return variant, base[0], variant
 246     base, *variant = client_name.split('_', 1)
 247     return client_name, base, variant[0] if variant else None
 248
 249
 250 def build_innertube_clients():
 251     THIRD_PARTY = {
 252         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 253     }
 254     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 255     priority = qualities(BASE_CLIENTS[::-1])
 256
 257     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 258         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 259         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 260         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 261         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 262
 263         _, base_client, variant = _split_innertube_client(client)
 264         ytcfg['priority'] = 10 * priority(base_client)
 265
 266         if not variant:
 267             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 268             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 269             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 270             embedscreen['priority'] -= 3
 271         elif variant == 'embedded':
 272             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 273             ytcfg['priority'] -= 2
 274         else:
 275             ytcfg['priority'] -= 3
 276
 277
 278 build_innertube_clients()
 279
 280
 281 class BadgeType(enum.Enum):
 282     AVAILABILITY_UNLISTED = enum.auto()
 283     AVAILABILITY_PRIVATE = enum.auto()
 284     AVAILABILITY_PUBLIC = enum.auto()
 285     AVAILABILITY_PREMIUM = enum.auto()
 286     AVAILABILITY_SUBSCRIPTION = enum.auto()
 287     LIVE_NOW = enum.auto()
 288
 289
 290 class YoutubeBaseInfoExtractor(InfoExtractor):
 291     """Provide base functions for Youtube extractors"""
 292
 293     _RESERVED_NAMES = (
 294         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 295         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 296         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 297         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 298
 299     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 300
 301     # _NETRC_MACHINE = 'youtube'
 302
 303     # If True it will raise an error if no login info is provided
 304     _LOGIN_REQUIRED = False
 305
 306     _INVIDIOUS_SITES = (
 307         # invidious-redirect websites
 308         r'(?:www\.)?redirect\.invidious\.io',
 309         r'(?:(?:www|dev)\.)?invidio\.us',
 310         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 311         r'(?:www\.)?invidious\.pussthecat\.org',
 312         r'(?:www\.)?invidious\.zee\.li',
 313         r'(?:www\.)?invidious\.ethibox\.fr',
 314         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 315         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 316         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 317         # youtube-dl invidious instances list
 318         r'(?:(?:www|no)\.)?invidiou\.sh',
 319         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 320         r'(?:www\.)?invidious\.kabi\.tk',
 321         r'(?:www\.)?invidious\.mastodon\.host',
 322         r'(?:www\.)?invidious\.zapashcanon\.fr',
 323         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 324         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 325         r'(?:www\.)?invidious\.himiko\.cloud',
 326         r'(?:www\.)?invidious\.reallyancient\.tech',
 327         r'(?:www\.)?invidious\.tube',
 328         r'(?:www\.)?invidiou\.site',
 329         r'(?:www\.)?invidious\.site',
 330         r'(?:www\.)?invidious\.xyz',
 331         r'(?:www\.)?invidious\.nixnet\.xyz',
 332         r'(?:www\.)?invidious\.048596\.xyz',
 333         r'(?:www\.)?invidious\.drycat\.fr',
 334         r'(?:www\.)?inv\.skyn3t\.in',
 335         r'(?:www\.)?tube\.poal\.co',
 336         r'(?:www\.)?tube\.connect\.cafe',
 337         r'(?:www\.)?vid\.wxzm\.sx',
 338         r'(?:www\.)?vid\.mint\.lgbt',
 339         r'(?:www\.)?vid\.puffyan\.us',
 340         r'(?:www\.)?yewtu\.be',
 341         r'(?:www\.)?yt\.elukerio\.org',
 342         r'(?:www\.)?yt\.lelux\.fi',
 343         r'(?:www\.)?invidious\.ggc-project\.de',
 344         r'(?:www\.)?yt\.maisputain\.ovh',
 345         r'(?:www\.)?ytprivate\.com',
 346         r'(?:www\.)?invidious\.13ad\.de',
 347         r'(?:www\.)?invidious\.toot\.koeln',
 348         r'(?:www\.)?invidious\.fdn\.fr',
 349         r'(?:www\.)?watch\.nettohikari\.com',
 350         r'(?:www\.)?invidious\.namazso\.eu',
 351         r'(?:www\.)?invidious\.silkky\.cloud',
 352         r'(?:www\.)?invidious\.exonip\.de',
 353         r'(?:www\.)?invidious\.riverside\.rocks',
 354         r'(?:www\.)?invidious\.blamefran\.net',
 355         r'(?:www\.)?invidious\.moomoo\.de',
 356         r'(?:www\.)?ytb\.trom\.tf',
 357         r'(?:www\.)?yt\.cyberhost\.uk',
 358         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 359         r'(?:www\.)?qklhadlycap4cnod\.onion',
 360         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 361         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 362         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 363         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 364         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 365         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 366         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 367         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 368         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 369         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 370         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 371         r'(?:www\.)?piped\.kavin\.rocks',
 372         r'(?:www\.)?piped\.silkky\.cloud',
 373         r'(?:www\.)?piped\.tokhmi\.xyz',
 374         r'(?:www\.)?piped\.moomoo\.me',
 375         r'(?:www\.)?il\.ax',
 376         r'(?:www\.)?piped\.syncpundit\.com',
 377         r'(?:www\.)?piped\.mha\.fi',
 378         r'(?:www\.)?piped\.mint\.lgbt',
 379         r'(?:www\.)?piped\.privacy\.com\.de',
 380     )
 381
 382     # extracted from account/account_menu ep
 383     # XXX: These are the supported YouTube UI and API languages,
 384     # which is slightly different from languages supported for translation in YouTube studio
 385     _SUPPORTED_LANG_CODES = [
 386         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 387         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 388         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 389         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 390         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 391         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 392     ]
 393
 394     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 395
 396     @functools.cached_property
 397     def _preferred_lang(self):
 398         """
 399         Returns a language code supported by YouTube for the user preferred language.
 400         Returns None if no preferred language set.
 401         """
 402         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 403         if not preferred_lang:
 404             return
 405         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 406             raise ExtractorError(
 407                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 408                 expected=True)
 409         elif preferred_lang != 'en':
 410             self.report_warning(
 411                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 412         return preferred_lang
 413
 414     def _initialize_consent(self):
 415         cookies = self._get_cookies('https://www.youtube.com/')
 416         if cookies.get('__Secure-3PSID'):
 417             return
 418         consent_id = None
 419         consent = cookies.get('CONSENT')
 420         if consent:
 421             if 'YES' in consent.value:
 422                 return
 423             consent_id = self._search_regex(
 424                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 425         if not consent_id:
 426             consent_id = random.randint(100, 999)
 427         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 428
 429     def _initialize_pref(self):
 430         cookies = self._get_cookies('https://www.youtube.com/')
 431         pref_cookie = cookies.get('PREF')
 432         pref = {}
 433         if pref_cookie:
 434             try:
 435                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 436             except ValueError:
 437                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 438         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 439         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 440
 441     def _real_initialize(self):
 442         self._initialize_pref()
 443         self._initialize_consent()
 444         self._check_login_required()
 445
 446     def _check_login_required(self):
 447         if self._LOGIN_REQUIRED and not self._cookies_passed:
 448             self.raise_login_required('Login details are needed to download this content', method='cookies')
 449
 450     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 451     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 452
 453     def _get_default_ytcfg(self, client='web'):
 454         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 455
 456     def _get_innertube_host(self, client='web'):
 457         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 458
 459     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 460         # try_get but with fallback to default ytcfg client values when present
 461         _func = lambda y: try_get(y, getter, expected_type)
 462         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 463
 464     def _extract_client_name(self, ytcfg, default_client='web'):
 465         return self._ytcfg_get_safe(
 466             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 467                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 468
 469     def _extract_client_version(self, ytcfg, default_client='web'):
 470         return self._ytcfg_get_safe(
 471             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 472                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 473
 474     def _select_api_hostname(self, req_api_hostname, default_client=None):
 475         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 476                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 477
 478     def _extract_api_key(self, ytcfg=None, default_client='web'):
 479         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 480
 481     def _extract_context(self, ytcfg=None, default_client='web'):
 482         context = get_first(
 483             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 484         # Enforce language and tz for extraction
 485         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 486         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 487         return context
 488
 489     _SAPISID = None
 490
 491     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 492         time_now = round(time.time())
 493         if self._SAPISID is None:
 494             yt_cookies = self._get_cookies('https://www.youtube.com')
 495             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 496             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 497             sapisid_cookie = dict_get(
 498                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 499             if sapisid_cookie and sapisid_cookie.value:
 500                 self._SAPISID = sapisid_cookie.value
 501                 self.write_debug('Extracted SAPISID cookie')
 502                 # SAPISID cookie is required if not already present
 503                 if not yt_cookies.get('SAPISID'):
 504                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 505                     self._set_cookie(
 506                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 507             else:
 508                 self._SAPISID = False
 509         if not self._SAPISID:
 510             return None
 511         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 512         sapisidhash = hashlib.sha1(
 513             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 514         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 515
 516     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 517                   note='Downloading API JSON', errnote='Unable to download API page',
 518                   context=None, api_key=None, api_hostname=None, default_client='web'):
 519
 520         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 521         data.update(query)
 522         real_headers = self.generate_api_headers(default_client=default_client)
 523         real_headers.update({'content-type': 'application/json'})
 524         if headers:
 525             real_headers.update(headers)
 526         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 527                    or api_key or self._extract_api_key(default_client=default_client))
 528         return self._download_json(
 529             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 530             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 531             data=json.dumps(data).encode('utf8'), headers=real_headers,
 532             query={'key': api_key, 'prettyPrint': 'false'})
 533
 534     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 535         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 536
 537     @staticmethod
 538     def _extract_session_index(*data):
 539         """
 540         Index of current account in account list.
 541         See: https://github.com/yt-dlp/yt-dlp/pull/519
 542         """
 543         for ytcfg in data:
 544             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 545             if session_index is not None:
 546                 return session_index
 547
 548     # Deprecated?
 549     def _extract_identity_token(self, ytcfg=None, webpage=None):
 550         if ytcfg:
 551             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 552             if token:
 553                 return token
 554         if webpage:
 555             return self._search_regex(
 556                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 557                 'identity token', default=None, fatal=False)
 558
 559     @staticmethod
 560     def _extract_account_syncid(*args):
 561         """
 562         Extract syncId required to download private playlists of secondary channels
 563         @params response and/or ytcfg
 564         """
 565         for data in args:
 566             # ytcfg includes channel_syncid if on secondary channel
 567             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 568             if delegated_sid:
 569                 return delegated_sid
 570             sync_ids = (try_get(
 571                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 572                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 573             if len(sync_ids) >= 2 and sync_ids[1]:
 574                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 575                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 576                 return sync_ids[0]
 577
 578     @staticmethod
 579     def _extract_visitor_data(*args):
 580         """
 581         Extracts visitorData from an API response or ytcfg
 582         Appears to be used to track session state
 583         """
 584         return get_first(
 585             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 586             expected_type=str)
 587
 588     @functools.cached_property
 589     def is_authenticated(self):
 590         return bool(self._generate_sapisidhash_header())
 591
 592     def extract_ytcfg(self, video_id, webpage):
 593         if not webpage:
 594             return {}
 595         return self._parse_json(
 596             self._search_regex(
 597                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 598                 default='{}'), video_id, fatal=False) or {}
 599
 600     def generate_api_headers(
 601             self, *, ytcfg=None, account_syncid=None, session_index=None,
 602             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 603
 604         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 605         headers = {
 606             'X-YouTube-Client-Name': str(
 607                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 608             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 609             'Origin': origin,
 610             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 611             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 612             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 613             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 614         }
 615         if session_index is None:
 616             session_index = self._extract_session_index(ytcfg)
 617         if account_syncid or session_index is not None:
 618             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 619
 620         auth = self._generate_sapisidhash_header(origin)
 621         if auth is not None:
 622             headers['Authorization'] = auth
 623             headers['X-Origin'] = origin
 624         return filter_dict(headers)
 625
 626     def _download_ytcfg(self, client, video_id):
 627         url = {
 628             'web': 'https://www.youtube.com',
 629             'web_music': 'https://music.youtube.com',
 630             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 631         }.get(client)
 632         if not url:
 633             return {}
 634         webpage = self._download_webpage(
 635             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 636         return self.extract_ytcfg(video_id, webpage) or {}
 637
 638     @staticmethod
 639     def _build_api_continuation_query(continuation, ctp=None):
 640         query = {
 641             'continuation': continuation
 642         }
 643         # TODO: Inconsistency with clickTrackingParams.
 644         # Currently we have a fixed ctp contained within context (from ytcfg)
 645         # and a ctp in root query for continuation.
 646         if ctp:
 647             query['clickTracking'] = {'clickTrackingParams': ctp}
 648         return query
 649
 650     @classmethod
 651     def _extract_next_continuation_data(cls, renderer):
 652         next_continuation = try_get(
 653             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 654                        lambda x: x['continuation']['reloadContinuationData']), dict)
 655         if not next_continuation:
 656             return
 657         continuation = next_continuation.get('continuation')
 658         if not continuation:
 659             return
 660         ctp = next_continuation.get('clickTrackingParams')
 661         return cls._build_api_continuation_query(continuation, ctp)
 662
 663     @classmethod
 664     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 665         if isinstance(continuation_ep, dict):
 666             continuation = try_get(
 667                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 668             if not continuation:
 669                 return
 670             ctp = continuation_ep.get('clickTrackingParams')
 671             return cls._build_api_continuation_query(continuation, ctp)
 672
 673     @classmethod
 674     def _extract_continuation(cls, renderer):
 675         next_continuation = cls._extract_next_continuation_data(renderer)
 676         if next_continuation:
 677             return next_continuation
 678
 679         return traverse_obj(renderer, (
 680             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 681             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 682         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 683
 684     @classmethod
 685     def _extract_alerts(cls, data):
 686         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 687             if not isinstance(alert_dict, dict):
 688                 continue
 689             for alert in alert_dict.values():
 690                 alert_type = alert.get('type')
 691                 if not alert_type:
 692                     continue
 693                 message = cls._get_text(alert, 'text')
 694                 if message:
 695                     yield alert_type, message
 696
 697     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 698         errors, warnings = [], []
 699         for alert_type, alert_message in alerts:
 700             if alert_type.lower() == 'error' and fatal:
 701                 errors.append([alert_type, alert_message])
 702             elif alert_message not in self._IGNORED_WARNINGS:
 703                 warnings.append([alert_type, alert_message])
 704
 705         for alert_type, alert_message in (warnings + errors[:-1]):
 706             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 707         if errors:
 708             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 709
 710     def _extract_and_report_alerts(self, data, *args, **kwargs):
 711         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 712
 713     def _extract_badges(self, renderer: dict):
 714         privacy_icon_map = {
 715             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 716             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 717             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 718         }
 719
 720         badge_style_map = {
 721             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 722             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 723             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 724         }
 725
 726         label_map = {
 727             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 728             'private': BadgeType.AVAILABILITY_PRIVATE,
 729             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 730             'live': BadgeType.LIVE_NOW,
 731             'premium': BadgeType.AVAILABILITY_PREMIUM
 732         }
 733
 734         badges = []
 735         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer'), default=[]):
 736             badge_type = (
 737                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 738                 or badge_style_map.get(traverse_obj(badge, 'style'))
 739             )
 740             if badge_type:
 741                 badges.append({'type': badge_type})
 742                 continue
 743
 744             # fallback, won't work in some languages
 745             label = traverse_obj(badge, 'label', expected_type=str, default='')
 746             for match, label_badge_type in label_map.items():
 747                 if match in label.lower():
 748                     badges.append({'type': badge_type})
 749                     continue
 750
 751         return badges
 752
 753     @staticmethod
 754     def _has_badge(badges, badge_type):
 755         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 756
 757     @staticmethod
 758     def _get_text(data, *path_list, max_runs=None):
 759         for path in path_list or [None]:
 760             if path is None:
 761                 obj = [data]
 762             else:
 763                 obj = traverse_obj(data, path, default=[])
 764                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 765                     obj = [obj]
 766             for item in obj:
 767                 text = try_get(item, lambda x: x['simpleText'], str)
 768                 if text:
 769                     return text
 770                 runs = try_get(item, lambda x: x['runs'], list) or []
 771                 if not runs and isinstance(item, list):
 772                     runs = item
 773
 774                 runs = runs[:min(len(runs), max_runs or len(runs))]
 775                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 776                 if text:
 777                     return text
 778
 779     def _get_count(self, data, *path_list):
 780         count_text = self._get_text(data, *path_list) or ''
 781         count = parse_count(count_text)
 782         if count is None:
 783             count = str_to_int(
 784                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 785         return count
 786
 787     @staticmethod
 788     def _extract_thumbnails(data, *path_list):
 789         """
 790         Extract thumbnails from thumbnails dict
 791         @param path_list: path list to level that contains 'thumbnails' key
 792         """
 793         thumbnails = []
 794         for path in path_list or [()]:
 795             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 796                 thumbnail_url = url_or_none(thumbnail.get('url'))
 797                 if not thumbnail_url:
 798                     continue
 799                 # Sometimes youtube gives a wrong thumbnail URL. See:
 800                 # https://github.com/yt-dlp/yt-dlp/issues/233
 801                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 802                 if 'maxresdefault' in thumbnail_url:
 803                     thumbnail_url = thumbnail_url.split('?')[0]
 804                 thumbnails.append({
 805                     'url': thumbnail_url,
 806                     'height': int_or_none(thumbnail.get('height')),
 807                     'width': int_or_none(thumbnail.get('width')),
 808                 })
 809         return thumbnails
 810
 811     @staticmethod
 812     def extract_relative_time(relative_time_text):
 813         """
 814         Extracts a relative time from string and converts to dt object
 815         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 816         """
 817         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 818         if mobj:
 819             start = mobj.group('start')
 820             if start:
 821                 return datetime_from_str(start)
 822             try:
 823                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 824             except ValueError:
 825                 return None
 826
 827     def _parse_time_text(self, text):
 828         if not text:
 829             return
 830         dt = self.extract_relative_time(text)
 831         timestamp = None
 832         if isinstance(dt, datetime.datetime):
 833             timestamp = calendar.timegm(dt.timetuple())
 834
 835         if timestamp is None:
 836             timestamp = (
 837                 unified_timestamp(text) or unified_timestamp(
 838                     self._search_regex(
 839                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 840                         text.lower(), 'time text', default=None)))
 841
 842         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 843             self.report_warning(
 844                 f'Cannot parse localized time text "{text}"', only_once=True)
 845         return timestamp
 846
 847     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 848                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 849                           default_client='web'):
 850         for retry in self.RetryManager():
 851             try:
 852                 response = self._call_api(
 853                     ep=ep, fatal=True, headers=headers,
 854                     video_id=item_id, query=query, note=note,
 855                     context=self._extract_context(ytcfg, default_client),
 856                     api_key=self._extract_api_key(ytcfg, default_client),
 857                     api_hostname=api_hostname, default_client=default_client)
 858             except ExtractorError as e:
 859                 if not isinstance(e.cause, network_exceptions):
 860                     return self._error_or_warning(e, fatal=fatal)
 861                 elif not isinstance(e.cause, urllib.error.HTTPError):
 862                     retry.error = e
 863                     continue
 864
 865                 first_bytes = e.cause.read(512)
 866                 if not is_html(first_bytes):
 867                     yt_error = try_get(
 868                         self._parse_json(
 869                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 870                         lambda x: x['error']['message'], str)
 871                     if yt_error:
 872                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 873                 # Downloading page may result in intermittent 5xx HTTP error
 874                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 875                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 876                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 877                 if e.cause.code not in (403, 429):
 878                     retry.error = e
 879                     continue
 880                 return self._error_or_warning(e, fatal=fatal)
 881
 882             try:
 883                 self._extract_and_report_alerts(response, only_once=True)
 884             except ExtractorError as e:
 885                 # YouTube servers may return errors we want to retry on in a 200 OK response
 886                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 887                 if 'unknown error' in e.msg.lower():
 888                     retry.error = e
 889                     continue
 890                 return self._error_or_warning(e, fatal=fatal)
 891             # Youtube sometimes sends incomplete data
 892             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 893             if not traverse_obj(response, *variadic(check_get_keys)):
 894                 retry.error = ExtractorError('Incomplete data received', expected=True)
 895                 continue
 896
 897             return response
 898
 899     @staticmethod
 900     def is_music_url(url):
 901         return re.match(r'https?://music\.youtube\.com/', url) is not None
 902
 903     def _extract_video(self, renderer):
 904         video_id = renderer.get('videoId')
 905         title = self._get_text(renderer, 'title')
 906         description = self._get_text(renderer, 'descriptionSnippet')
 907
 908         duration = int_or_none(renderer.get('lengthSeconds'))
 909         if duration is None:
 910             duration = parse_duration(self._get_text(
 911                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 912         if duration is None:
 913             duration = parse_duration(self._search_regex(
 914                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 915                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 916                 video_id, default=None, group='duration'))
 917
 918         # videoInfo is a string like '50K views • 10 years ago'.
 919         view_count = self._get_count(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo')
 920         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 921         channel_id = traverse_obj(
 922             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 923             expected_type=str, get_all=False)
 924         time_text = self._get_text(renderer, 'publishedTimeText', 'videoInfo') or ''
 925         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 926         overlay_style = traverse_obj(
 927             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 928             get_all=False, expected_type=str)
 929         badges = self._extract_badges(renderer)
 930         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 931         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 932             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 933             expected_type=str)) or ''
 934         url = f'https://www.youtube.com/watch?v={video_id}'
 935         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 936             url = f'https://www.youtube.com/shorts/{video_id}'
 937
 938         live_status = (
 939             'is_upcoming' if scheduled_timestamp is not None
 940             else 'was_live' if 'streamed' in time_text.lower()
 941             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
 942             else None)
 943
 944         return {
 945             '_type': 'url',
 946             'ie_key': YoutubeIE.ie_key(),
 947             'id': video_id,
 948             'url': url,
 949             'title': title,
 950             'description': description,
 951             'duration': duration,
 952             'uploader': uploader,
 953             'channel_id': channel_id,
 954             'thumbnails': thumbnails,
 955             'timestamp': (self._parse_time_text(time_text)
 956                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
 957                           else None),
 958             'release_timestamp': scheduled_timestamp,
 959             'availability':
 960                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
 961                 else self._availability(
 962                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
 963                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
 964                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
 965                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
 966             'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count': view_count,
 967             'live_status': live_status
 968         }
 969
 970
 971 class YoutubeIE(YoutubeBaseInfoExtractor):
 972     IE_DESC = 'YouTube'
 973     _VALID_URL = r"""(?x)^
 974                      (
 975                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 976                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 977                             (?:www\.)?deturl\.com/www\.youtube\.com|
 978                             (?:www\.)?pwnyoutube\.com|
 979                             (?:www\.)?hooktube\.com|
 980                             (?:www\.)?yourepeat\.com|
 981                             tube\.majestyc\.net|
 982                             %(invidious)s|
 983                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 984                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 985                          (?:                                                  # the various things that can precede the ID:
 986                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 987                              |(?:                                             # or the v= param in all its forms
 988                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 989                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 990                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 991                                  v=
 992                              )
 993                          ))
 994                          |(?:
 995                             youtu\.be|                                        # just youtu.be/xxxx
 996                             vid\.plus|                                        # or vid.plus/xxxx
 997                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 998                             %(invidious)s
 999                          )/
1000                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1001                          )
1002                      )?                                                       # all until now is optional -> you can pass the naked ID
1003                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1004                      (?(1).+)?                                                # if we found the ID, everything can follow
1005                      (?:\#|$)""" % {
1006         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1007     }
1008     _EMBED_REGEX = [
1009         r'''(?x)
1010             (?:
1011                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1012                 data-video-url=|
1013                 <embed[^>]+?src=|
1014                 embedSWF\(?:\s*|
1015                 <object[^>]+data=|
1016                 new\s+SWFObject\(
1017             )
1018             (["\'])
1019                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1020                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1021             \1''',
1022         # https://wordpress.org/plugins/lazy-load-for-videos/
1023         r'''(?xs)
1024             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1025             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1026     ]
1027
1028     _PLAYER_INFO_RE = (
1029         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1030         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1031         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1032     )
1033     _formats = {
1034         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1035         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1036         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1037         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1038         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1039         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1040         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1041         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1042         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1043         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1044         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1045         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1046         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1047         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1048         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1049         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1050         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1051         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1052
1053
1054         # 3D videos
1055         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1056         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1057         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1058         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1059         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1060         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1061         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1062
1063         # Apple HTTP Live Streaming
1064         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1065         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1066         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1067         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1068         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1069         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1070         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1071         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1072
1073         # DASH mp4 video
1074         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1075         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1076         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1077         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1078         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1079         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1080         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1081         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1082         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1083         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1084         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1085         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1086
1087         # Dash mp4 audio
1088         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1089         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1090         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1091         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1092         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1093         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1094         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1095
1096         # Dash webm
1097         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1098         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1099         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1100         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1101         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1102         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1103         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1104         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1105         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1106         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1107         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1108         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1109         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1110         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1111         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1112         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1113         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1114         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1115         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1116         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1117         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1118         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1119
1120         # Dash webm audio
1121         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1122         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1123
1124         # Dash webm audio with opus inside
1125         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1126         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1127         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1128
1129         # RTMP (unnamed)
1130         '_rtmp': {'protocol': 'rtmp'},
1131
1132         # av01 video only formats sometimes served with "unknown" codecs
1133         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1134         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1135         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1136         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1137         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1138         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1139         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1140         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1141     }
1142     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1143
1144     _GEO_BYPASS = False
1145
1146     IE_NAME = 'youtube'
1147     _TESTS = [
1148         {
1149             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1150             'info_dict': {
1151                 'id': 'BaW_jenozKc',
1152                 'ext': 'mp4',
1153                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1154                 'uploader': 'Philipp Hagemeister',
1155                 'uploader_id': 'phihag',
1156                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1157                 'channel': 'Philipp Hagemeister',
1158                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1159                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1160                 'upload_date': '20121002',
1161                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1162                 'categories': ['Science & Technology'],
1163                 'tags': ['youtube-dl'],
1164                 'duration': 10,
1165                 'view_count': int,
1166                 'like_count': int,
1167                 'availability': 'public',
1168                 'playable_in_embed': True,
1169                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1170                 'live_status': 'not_live',
1171                 'age_limit': 0,
1172                 'start_time': 1,
1173                 'end_time': 9,
1174                 'comment_count': int,
1175                 'channel_follower_count': int
1176             }
1177         },
1178         {
1179             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1180             'note': 'Embed-only video (#1746)',
1181             'info_dict': {
1182                 'id': 'yZIXLfi8CZQ',
1183                 'ext': 'mp4',
1184                 'upload_date': '20120608',
1185                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1186                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1187                 'uploader': 'SET India',
1188                 'uploader_id': 'setindia',
1189                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1190                 'age_limit': 18,
1191             },
1192             'skip': 'Private video',
1193         },
1194         {
1195             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1196             'note': 'Use the first video ID in the URL',
1197             'info_dict': {
1198                 'id': 'BaW_jenozKc',
1199                 'ext': 'mp4',
1200                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1201                 'uploader': 'Philipp Hagemeister',
1202                 'uploader_id': 'phihag',
1203                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1204                 'channel': 'Philipp Hagemeister',
1205                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1206                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1207                 'upload_date': '20121002',
1208                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1209                 'categories': ['Science & Technology'],
1210                 'tags': ['youtube-dl'],
1211                 'duration': 10,
1212                 'view_count': int,
1213                 'like_count': int,
1214                 'availability': 'public',
1215                 'playable_in_embed': True,
1216                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1217                 'live_status': 'not_live',
1218                 'age_limit': 0,
1219                 'comment_count': int,
1220                 'channel_follower_count': int
1221             },
1222             'params': {
1223                 'skip_download': True,
1224             },
1225         },
1226         {
1227             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1228             'note': '256k DASH audio (format 141) via DASH manifest',
1229             'info_dict': {
1230                 'id': 'a9LDPn-MO4I',
1231                 'ext': 'm4a',
1232                 'upload_date': '20121002',
1233                 'uploader_id': '8KVIDEO',
1234                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1235                 'description': '',
1236                 'uploader': '8KVIDEO',
1237                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1238             },
1239             'params': {
1240                 'youtube_include_dash_manifest': True,
1241                 'format': '141',
1242             },
1243             'skip': 'format 141 not served anymore',
1244         },
1245         # DASH manifest with encrypted signature
1246         {
1247             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1248             'info_dict': {
1249                 'id': 'IB3lcPjvWLA',
1250                 'ext': 'm4a',
1251                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1252                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1253                 'duration': 244,
1254                 'uploader': 'AfrojackVEVO',
1255                 'uploader_id': 'AfrojackVEVO',
1256                 'upload_date': '20131011',
1257                 'abr': 129.495,
1258                 'like_count': int,
1259                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1260                 'playable_in_embed': True,
1261                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1262                 'view_count': int,
1263                 'track': 'The Spark',
1264                 'live_status': 'not_live',
1265                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1266                 'channel': 'Afrojack',
1267                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1268                 'tags': 'count:19',
1269                 'availability': 'public',
1270                 'categories': ['Music'],
1271                 'age_limit': 0,
1272                 'alt_title': 'The Spark',
1273                 'channel_follower_count': int
1274             },
1275             'params': {
1276                 'youtube_include_dash_manifest': True,
1277                 'format': '141/bestaudio[ext=m4a]',
1278             },
1279         },
1280         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1281         {
1282             'note': 'Embed allowed age-gate video',
1283             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1284             'info_dict': {
1285                 'id': 'HtVdAasjOgU',
1286                 'ext': 'mp4',
1287                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1288                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1289                 'duration': 142,
1290                 'uploader': 'The Witcher',
1291                 'uploader_id': 'WitcherGame',
1292                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1293                 'upload_date': '20140605',
1294                 'age_limit': 18,
1295                 'categories': ['Gaming'],
1296                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1297                 'availability': 'needs_auth',
1298                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1299                 'like_count': int,
1300                 'channel': 'The Witcher',
1301                 'live_status': 'not_live',
1302                 'tags': 'count:17',
1303                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1304                 'playable_in_embed': True,
1305                 'view_count': int,
1306                 'channel_follower_count': int
1307             },
1308         },
1309         {
1310             'note': 'Age-gate video with embed allowed in public site',
1311             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1312             'info_dict': {
1313                 'id': 'HsUATh_Nc2U',
1314                 'ext': 'mp4',
1315                 'title': 'Godzilla 2 (Official Video)',
1316                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1317                 'upload_date': '20200408',
1318                 'uploader_id': 'FlyingKitty900',
1319                 'uploader': 'FlyingKitty',
1320                 'age_limit': 18,
1321                 'availability': 'needs_auth',
1322                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1323                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1324                 'channel': 'FlyingKitty',
1325                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1326                 'view_count': int,
1327                 'categories': ['Entertainment'],
1328                 'live_status': 'not_live',
1329                 'tags': ['Flyingkitty', 'godzilla 2'],
1330                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1331                 'like_count': int,
1332                 'duration': 177,
1333                 'playable_in_embed': True,
1334                 'channel_follower_count': int
1335             },
1336         },
1337         {
1338             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1339             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1340             'info_dict': {
1341                 'id': 'Tq92D6wQ1mg',
1342                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1343                 'ext': 'mp4',
1344                 'upload_date': '20191228',
1345                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1346                 'uploader': 'Projekt Melody',
1347                 'description': 'md5:17eccca93a786d51bc67646756894066',
1348                 'age_limit': 18,
1349                 'like_count': int,
1350                 'availability': 'needs_auth',
1351                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1352                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1353                 'view_count': int,
1354                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1355                 'channel': 'Projekt Melody',
1356                 'live_status': 'not_live',
1357                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1358                 'playable_in_embed': True,
1359                 'categories': ['Entertainment'],
1360                 'duration': 106,
1361                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1362                 'comment_count': int,
1363                 'channel_follower_count': int
1364             },
1365         },
1366         {
1367             'note': 'Non-Agegated non-embeddable video',
1368             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1369             'info_dict': {
1370                 'id': 'MeJVWBSsPAY',
1371                 'ext': 'mp4',
1372                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1373                 'uploader': 'Herr Lurik',
1374                 'uploader_id': 'st3in234',
1375                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1376                 'upload_date': '20130730',
1377                 'track': 'Such mich find mich',
1378                 'age_limit': 0,
1379                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1380                 'like_count': int,
1381                 'playable_in_embed': False,
1382                 'creator': 'OOMPH!',
1383                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1384                 'view_count': int,
1385                 'alt_title': 'Such mich find mich',
1386                 'duration': 210,
1387                 'channel': 'Herr Lurik',
1388                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1389                 'categories': ['Music'],
1390                 'availability': 'public',
1391                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1392                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1393                 'live_status': 'not_live',
1394                 'artist': 'OOMPH!',
1395                 'channel_follower_count': int
1396             },
1397         },
1398         {
1399             'note': 'Non-bypassable age-gated video',
1400             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1401             'only_matching': True,
1402         },
1403         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1404         # YouTube Red ad is not captured for creator
1405         {
1406             'url': '__2ABJjxzNo',
1407             'info_dict': {
1408                 'id': '__2ABJjxzNo',
1409                 'ext': 'mp4',
1410                 'duration': 266,
1411                 'upload_date': '20100430',
1412                 'uploader_id': 'deadmau5',
1413                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1414                 'creator': 'deadmau5',
1415                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1416                 'uploader': 'deadmau5',
1417                 'title': 'Deadmau5 - Some Chords (HD)',
1418                 'alt_title': 'Some Chords',
1419                 'availability': 'public',
1420                 'tags': 'count:14',
1421                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1422                 'view_count': int,
1423                 'live_status': 'not_live',
1424                 'channel': 'deadmau5',
1425                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1426                 'like_count': int,
1427                 'track': 'Some Chords',
1428                 'artist': 'deadmau5',
1429                 'playable_in_embed': True,
1430                 'age_limit': 0,
1431                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1432                 'categories': ['Music'],
1433                 'album': 'Some Chords',
1434                 'channel_follower_count': int
1435             },
1436             'expected_warnings': [
1437                 'DASH manifest missing',
1438             ]
1439         },
1440         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1441         {
1442             'url': 'lqQg6PlCWgI',
1443             'info_dict': {
1444                 'id': 'lqQg6PlCWgI',
1445                 'ext': 'mp4',
1446                 'duration': 6085,
1447                 'upload_date': '20150827',
1448                 'uploader_id': 'olympic',
1449                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1450                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1451                 'uploader': 'Olympics',
1452                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1453                 'like_count': int,
1454                 'release_timestamp': 1343767800,
1455                 'playable_in_embed': True,
1456                 'categories': ['Sports'],
1457                 'release_date': '20120731',
1458                 'channel': 'Olympics',
1459                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1460                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1461                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1462                 'age_limit': 0,
1463                 'availability': 'public',
1464                 'live_status': 'was_live',
1465                 'view_count': int,
1466                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1467                 'channel_follower_count': int
1468             },
1469             'params': {
1470                 'skip_download': 'requires avconv',
1471             }
1472         },
1473         # Non-square pixels
1474         {
1475             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1476             'info_dict': {
1477                 'id': '_b-2C3KPAM0',
1478                 'ext': 'mp4',
1479                 'stretched_ratio': 16 / 9.,
1480                 'duration': 85,
1481                 'upload_date': '20110310',
1482                 'uploader_id': 'AllenMeow',
1483                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1484                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1485                 'uploader': '孫ᄋᄅ',
1486                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1487                 'playable_in_embed': True,
1488                 'channel': '孫ᄋᄅ',
1489                 'age_limit': 0,
1490                 'tags': 'count:11',
1491                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1492                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1493                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1494                 'view_count': int,
1495                 'categories': ['People & Blogs'],
1496                 'like_count': int,
1497                 'live_status': 'not_live',
1498                 'availability': 'unlisted',
1499                 'comment_count': int,
1500                 'channel_follower_count': int
1501             },
1502         },
1503         # url_encoded_fmt_stream_map is empty string
1504         {
1505             'url': 'qEJwOuvDf7I',
1506             'info_dict': {
1507                 'id': 'qEJwOuvDf7I',
1508                 'ext': 'webm',
1509                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1510                 'description': '',
1511                 'upload_date': '20150404',
1512                 'uploader_id': 'spbelect',
1513                 'uploader': 'Наблюдатели Петербурга',
1514             },
1515             'params': {
1516                 'skip_download': 'requires avconv',
1517             },
1518             'skip': 'This live event has ended.',
1519         },
1520         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1521         {
1522             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1523             'info_dict': {
1524                 'id': 'FIl7x6_3R5Y',
1525                 'ext': 'webm',
1526                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1527                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1528                 'duration': 220,
1529                 'upload_date': '20150625',
1530                 'uploader_id': 'dorappi2000',
1531                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1532                 'uploader': 'dorappi2000',
1533                 'formats': 'mincount:31',
1534             },
1535             'skip': 'not actual anymore',
1536         },
1537         # DASH manifest with segment_list
1538         {
1539             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1540             'md5': '8ce563a1d667b599d21064e982ab9e31',
1541             'info_dict': {
1542                 'id': 'CsmdDsKjzN8',
1543                 'ext': 'mp4',
1544                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1545                 'uploader': 'Airtek',
1546                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1547                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1548                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1549             },
1550             'params': {
1551                 'youtube_include_dash_manifest': True,
1552                 'format': '135',  # bestvideo
1553             },
1554             'skip': 'This live event has ended.',
1555         },
1556         {
1557             # Multifeed videos (multiple cameras), URL is for Main Camera
1558             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1559             'info_dict': {
1560                 'id': 'jvGDaLqkpTg',
1561                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1562                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1563             },
1564             'playlist': [{
1565                 'info_dict': {
1566                     'id': 'jvGDaLqkpTg',
1567                     'ext': 'mp4',
1568                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1569                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1570                     'duration': 10643,
1571                     'upload_date': '20161111',
1572                     'uploader': 'Team PGP',
1573                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1574                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1575                 },
1576             }, {
1577                 'info_dict': {
1578                     'id': '3AKt1R1aDnw',
1579                     'ext': 'mp4',
1580                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1581                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1582                     'duration': 10991,
1583                     'upload_date': '20161111',
1584                     'uploader': 'Team PGP',
1585                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1586                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1587                 },
1588             }, {
1589                 'info_dict': {
1590                     'id': 'RtAMM00gpVc',
1591                     'ext': 'mp4',
1592                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1593                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1594                     'duration': 10995,
1595                     'upload_date': '20161111',
1596                     'uploader': 'Team PGP',
1597                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1598                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1599                 },
1600             }, {
1601                 'info_dict': {
1602                     'id': '6N2fdlP3C5U',
1603                     'ext': 'mp4',
1604                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1605                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1606                     'duration': 10990,
1607                     'upload_date': '20161111',
1608                     'uploader': 'Team PGP',
1609                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1610                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1611                 },
1612             }],
1613             'params': {
1614                 'skip_download': True,
1615             },
1616             'skip': 'Not multifeed anymore',
1617         },
1618         {
1619             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1620             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1621             'info_dict': {
1622                 'id': 'gVfLd0zydlo',
1623                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1624             },
1625             'playlist_count': 2,
1626             'skip': 'Not multifeed anymore',
1627         },
1628         {
1629             'url': 'https://vid.plus/FlRa-iH7PGw',
1630             'only_matching': True,
1631         },
1632         {
1633             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1634             'only_matching': True,
1635         },
1636         {
1637             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1638             # Also tests cut-off URL expansion in video description (see
1639             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1640             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1641             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1642             'info_dict': {
1643                 'id': 'lsguqyKfVQg',
1644                 'ext': 'mp4',
1645                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1646                 'alt_title': 'Dark Walk',
1647                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1648                 'duration': 133,
1649                 'upload_date': '20151119',
1650                 'uploader_id': 'IronSoulElf',
1651                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1652                 'uploader': 'IronSoulElf',
1653                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1654                 'track': 'Dark Walk',
1655                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1656                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1657                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1658                 'categories': ['Film & Animation'],
1659                 'view_count': int,
1660                 'live_status': 'not_live',
1661                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1662                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1663                 'tags': 'count:13',
1664                 'availability': 'public',
1665                 'channel': 'IronSoulElf',
1666                 'playable_in_embed': True,
1667                 'like_count': int,
1668                 'age_limit': 0,
1669                 'channel_follower_count': int
1670             },
1671             'params': {
1672                 'skip_download': True,
1673             },
1674         },
1675         {
1676             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1677             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1678             'only_matching': True,
1679         },
1680         {
1681             # Video with yt:stretch=17:0
1682             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1683             'info_dict': {
1684                 'id': 'Q39EVAstoRM',
1685                 'ext': 'mp4',
1686                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1687                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1688                 'upload_date': '20151107',
1689                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1690                 'uploader': 'CH GAMER DROID',
1691             },
1692             'params': {
1693                 'skip_download': True,
1694             },
1695             'skip': 'This video does not exist.',
1696         },
1697         {
1698             # Video with incomplete 'yt:stretch=16:'
1699             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1700             'only_matching': True,
1701         },
1702         {
1703             # Video licensed under Creative Commons
1704             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1705             'info_dict': {
1706                 'id': 'M4gD1WSo5mA',
1707                 'ext': 'mp4',
1708                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1709                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1710                 'duration': 721,
1711                 'upload_date': '20150128',
1712                 'uploader_id': 'BerkmanCenter',
1713                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1714                 'uploader': 'The Berkman Klein Center for Internet & Society',
1715                 'license': 'Creative Commons Attribution license (reuse allowed)',
1716                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1717                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1718                 'like_count': int,
1719                 'age_limit': 0,
1720                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1721                 'channel': 'The Berkman Klein Center for Internet & Society',
1722                 'availability': 'public',
1723                 'view_count': int,
1724                 'categories': ['Education'],
1725                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1726                 'live_status': 'not_live',
1727                 'playable_in_embed': True,
1728                 'comment_count': int,
1729                 'channel_follower_count': int,
1730                 'chapters': list,
1731             },
1732             'params': {
1733                 'skip_download': True,
1734             },
1735         },
1736         {
1737             # Channel-like uploader_url
1738             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1739             'info_dict': {
1740                 'id': 'eQcmzGIKrzg',
1741                 'ext': 'mp4',
1742                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1743                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1744                 'duration': 4060,
1745                 'upload_date': '20151120',
1746                 'uploader': 'Bernie Sanders',
1747                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1748                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1749                 'license': 'Creative Commons Attribution license (reuse allowed)',
1750                 'playable_in_embed': True,
1751                 'tags': 'count:12',
1752                 'like_count': int,
1753                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1754                 'age_limit': 0,
1755                 'availability': 'public',
1756                 'categories': ['News & Politics'],
1757                 'channel': 'Bernie Sanders',
1758                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1759                 'view_count': int,
1760                 'live_status': 'not_live',
1761                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1762                 'comment_count': int,
1763                 'channel_follower_count': int,
1764                 'chapters': list,
1765             },
1766             'params': {
1767                 'skip_download': True,
1768             },
1769         },
1770         {
1771             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1772             'only_matching': True,
1773         },
1774         {
1775             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1776             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1777             'only_matching': True,
1778         },
1779         {
1780             # Rental video preview
1781             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1782             'info_dict': {
1783                 'id': 'uGpuVWrhIzE',
1784                 'ext': 'mp4',
1785                 'title': 'Piku - Trailer',
1786                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1787                 'upload_date': '20150811',
1788                 'uploader': 'FlixMatrix',
1789                 'uploader_id': 'FlixMatrixKaravan',
1790                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1791                 'license': 'Standard YouTube License',
1792             },
1793             'params': {
1794                 'skip_download': True,
1795             },
1796             'skip': 'This video is not available.',
1797         },
1798         {
1799             # YouTube Red video with episode data
1800             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1801             'info_dict': {
1802                 'id': 'iqKdEhx-dD4',
1803                 'ext': 'mp4',
1804                 'title': 'Isolation - Mind Field (Ep 1)',
1805                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1806                 'duration': 2085,
1807                 'upload_date': '20170118',
1808                 'uploader': 'Vsauce',
1809                 'uploader_id': 'Vsauce',
1810                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1811                 'series': 'Mind Field',
1812                 'season_number': 1,
1813                 'episode_number': 1,
1814                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1815                 'tags': 'count:12',
1816                 'view_count': int,
1817                 'availability': 'public',
1818                 'age_limit': 0,
1819                 'channel': 'Vsauce',
1820                 'episode': 'Episode 1',
1821                 'categories': ['Entertainment'],
1822                 'season': 'Season 1',
1823                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1824                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1825                 'like_count': int,
1826                 'playable_in_embed': True,
1827                 'live_status': 'not_live',
1828                 'channel_follower_count': int
1829             },
1830             'params': {
1831                 'skip_download': True,
1832             },
1833             'expected_warnings': [
1834                 'Skipping DASH manifest',
1835             ],
1836         },
1837         {
1838             # The following content has been identified by the YouTube community
1839             # as inappropriate or offensive to some audiences.
1840             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1841             'info_dict': {
1842                 'id': '6SJNVb0GnPI',
1843                 'ext': 'mp4',
1844                 'title': 'Race Differences in Intelligence',
1845                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1846                 'duration': 965,
1847                 'upload_date': '20140124',
1848                 'uploader': 'New Century Foundation',
1849                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1850                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1851             },
1852             'params': {
1853                 'skip_download': True,
1854             },
1855             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1856         },
1857         {
1858             # itag 212
1859             'url': '1t24XAntNCY',
1860             'only_matching': True,
1861         },
1862         {
1863             # geo restricted to JP
1864             'url': 'sJL6WA-aGkQ',
1865             'only_matching': True,
1866         },
1867         {
1868             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1869             'only_matching': True,
1870         },
1871         {
1872             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1873             'only_matching': True,
1874         },
1875         {
1876             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1877             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1878             'only_matching': True,
1879         },
1880         {
1881             # DRM protected
1882             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1883             'only_matching': True,
1884         },
1885         {
1886             # Video with unsupported adaptive stream type formats
1887             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1888             'info_dict': {
1889                 'id': 'Z4Vy8R84T1U',
1890                 'ext': 'mp4',
1891                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1892                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1893                 'duration': 433,
1894                 'upload_date': '20130923',
1895                 'uploader': 'Amelia Putri Harwita',
1896                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1897                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1898                 'formats': 'maxcount:10',
1899             },
1900             'params': {
1901                 'skip_download': True,
1902                 'youtube_include_dash_manifest': False,
1903             },
1904             'skip': 'not actual anymore',
1905         },
1906         {
1907             # Youtube Music Auto-generated description
1908             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1909             'info_dict': {
1910                 'id': 'MgNrAu2pzNs',
1911                 'ext': 'mp4',
1912                 'title': 'Voyeur Girl',
1913                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1914                 'upload_date': '20190312',
1915                 'uploader': 'Stephen - Topic',
1916                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1917                 'artist': 'Stephen',
1918                 'track': 'Voyeur Girl',
1919                 'album': 'it\'s too much love to know my dear',
1920                 'release_date': '20190313',
1921                 'release_year': 2019,
1922                 'alt_title': 'Voyeur Girl',
1923                 'view_count': int,
1924                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1925                 'playable_in_embed': True,
1926                 'like_count': int,
1927                 'categories': ['Music'],
1928                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1929                 'channel': 'Stephen',
1930                 'availability': 'public',
1931                 'creator': 'Stephen',
1932                 'duration': 169,
1933                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1934                 'age_limit': 0,
1935                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1936                 'tags': 'count:11',
1937                 'live_status': 'not_live',
1938                 'channel_follower_count': int
1939             },
1940             'params': {
1941                 'skip_download': True,
1942             },
1943         },
1944         {
1945             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1946             'only_matching': True,
1947         },
1948         {
1949             # invalid -> valid video id redirection
1950             'url': 'DJztXj2GPfl',
1951             'info_dict': {
1952                 'id': 'DJztXj2GPfk',
1953                 'ext': 'mp4',
1954                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1955                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1956                 'upload_date': '20090125',
1957                 'uploader': 'Prochorowka',
1958                 'uploader_id': 'Prochorowka',
1959                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1960                 'artist': 'Panjabi MC',
1961                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1962                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1963             },
1964             'params': {
1965                 'skip_download': True,
1966             },
1967             'skip': 'Video unavailable',
1968         },
1969         {
1970             # empty description results in an empty string
1971             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1972             'info_dict': {
1973                 'id': 'x41yOUIvK2k',
1974                 'ext': 'mp4',
1975                 'title': 'IMG 3456',
1976                 'description': '',
1977                 'upload_date': '20170613',
1978                 'uploader_id': 'ElevageOrVert',
1979                 'uploader': 'ElevageOrVert',
1980                 'view_count': int,
1981                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1982                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1983                 'like_count': int,
1984                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1985                 'tags': [],
1986                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1987                 'availability': 'public',
1988                 'age_limit': 0,
1989                 'categories': ['Pets & Animals'],
1990                 'duration': 7,
1991                 'playable_in_embed': True,
1992                 'live_status': 'not_live',
1993                 'channel': 'ElevageOrVert',
1994                 'channel_follower_count': int
1995             },
1996             'params': {
1997                 'skip_download': True,
1998             },
1999         },
2000         {
2001             # with '};' inside yt initial data (see [1])
2002             # see [2] for an example with '};' inside ytInitialPlayerResponse
2003             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2004             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2005             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2006             'info_dict': {
2007                 'id': 'CHqg6qOn4no',
2008                 'ext': 'mp4',
2009                 'title': 'Part 77   Sort a list of simple types in c#',
2010                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2011                 'upload_date': '20130831',
2012                 'uploader_id': 'kudvenkat',
2013                 'uploader': 'kudvenkat',
2014                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2015                 'like_count': int,
2016                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2017                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2018                 'live_status': 'not_live',
2019                 'categories': ['Education'],
2020                 'availability': 'public',
2021                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2022                 'tags': 'count:12',
2023                 'playable_in_embed': True,
2024                 'age_limit': 0,
2025                 'view_count': int,
2026                 'duration': 522,
2027                 'channel': 'kudvenkat',
2028                 'comment_count': int,
2029                 'channel_follower_count': int,
2030                 'chapters': list,
2031             },
2032             'params': {
2033                 'skip_download': True,
2034             },
2035         },
2036         {
2037             # another example of '};' in ytInitialData
2038             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2039             'only_matching': True,
2040         },
2041         {
2042             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2043             'only_matching': True,
2044         },
2045         {
2046             # https://github.com/ytdl-org/youtube-dl/pull/28094
2047             'url': 'OtqTfy26tG0',
2048             'info_dict': {
2049                 'id': 'OtqTfy26tG0',
2050                 'ext': 'mp4',
2051                 'title': 'Burn Out',
2052                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2053                 'upload_date': '20141120',
2054                 'uploader': 'The Cinematic Orchestra - Topic',
2055                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2056                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2057                 'artist': 'The Cinematic Orchestra',
2058                 'track': 'Burn Out',
2059                 'album': 'Every Day',
2060                 'like_count': int,
2061                 'live_status': 'not_live',
2062                 'alt_title': 'Burn Out',
2063                 'duration': 614,
2064                 'age_limit': 0,
2065                 'view_count': int,
2066                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2067                 'creator': 'The Cinematic Orchestra',
2068                 'channel': 'The Cinematic Orchestra',
2069                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2070                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2071                 'availability': 'public',
2072                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2073                 'categories': ['Music'],
2074                 'playable_in_embed': True,
2075                 'channel_follower_count': int
2076             },
2077             'params': {
2078                 'skip_download': True,
2079             },
2080         },
2081         {
2082             # controversial video, only works with bpctr when authenticated with cookies
2083             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2084             'only_matching': True,
2085         },
2086         {
2087             # controversial video, requires bpctr/contentCheckOk
2088             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2089             'info_dict': {
2090                 'id': 'SZJvDhaSDnc',
2091                 'ext': 'mp4',
2092                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2093                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2094                 'uploader': 'CBS Mornings',
2095                 'uploader_id': 'CBSThisMorning',
2096                 'upload_date': '20140716',
2097                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2098                 'duration': 170,
2099                 'categories': ['News & Politics'],
2100                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2101                 'view_count': int,
2102                 'channel': 'CBS Mornings',
2103                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2104                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2105                 'age_limit': 18,
2106                 'availability': 'needs_auth',
2107                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2108                 'like_count': int,
2109                 'live_status': 'not_live',
2110                 'playable_in_embed': True,
2111                 'channel_follower_count': int
2112             }
2113         },
2114         {
2115             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2116             'url': 'cBvYw8_A0vQ',
2117             'info_dict': {
2118                 'id': 'cBvYw8_A0vQ',
2119                 'ext': 'mp4',
2120                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2121                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2122                 'upload_date': '20201120',
2123                 'uploader': 'Walk around Japan',
2124                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2125                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2126                 'duration': 1456,
2127                 'categories': ['Travel & Events'],
2128                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2129                 'view_count': int,
2130                 'channel': 'Walk around Japan',
2131                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2132                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2133                 'age_limit': 0,
2134                 'availability': 'public',
2135                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2136                 'live_status': 'not_live',
2137                 'playable_in_embed': True,
2138                 'channel_follower_count': int
2139             },
2140             'params': {
2141                 'skip_download': True,
2142             },
2143         }, {
2144             # Has multiple audio streams
2145             'url': 'WaOKSUlf4TM',
2146             'only_matching': True
2147         }, {
2148             # Requires Premium: has format 141 when requested using YTM url
2149             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2150             'only_matching': True
2151         }, {
2152             # multiple subtitles with same lang_code
2153             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2154             'only_matching': True,
2155         }, {
2156             # Force use android client fallback
2157             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2158             'info_dict': {
2159                 'id': 'YOelRv7fMxY',
2160                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2161                 'ext': '3gp',
2162                 'upload_date': '20210624',
2163                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2164                 'uploader': 'colinfurze',
2165                 'uploader_id': 'colinfurze',
2166                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2167                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2168                 'duration': 596,
2169                 'categories': ['Entertainment'],
2170                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2171                 'view_count': int,
2172                 'channel': 'colinfurze',
2173                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2174                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2175                 'age_limit': 0,
2176                 'availability': 'public',
2177                 'like_count': int,
2178                 'live_status': 'not_live',
2179                 'playable_in_embed': True,
2180                 'channel_follower_count': int,
2181                 'chapters': list,
2182             },
2183             'params': {
2184                 'format': '17',  # 3gp format available on android
2185                 'extractor_args': {'youtube': {'player_client': ['android']}},
2186             },
2187         },
2188         {
2189             # Skip download of additional client configs (remix client config in this case)
2190             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2191             'only_matching': True,
2192             'params': {
2193                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2194             },
2195         }, {
2196             # shorts
2197             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2198             'only_matching': True,
2199         }, {
2200             'note': 'Storyboards',
2201             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2202             'info_dict': {
2203                 'id': '5KLPxDtMqe8',
2204                 'ext': 'mhtml',
2205                 'format_id': 'sb0',
2206                 'title': 'Your Brain is Plastic',
2207                 'uploader_id': 'scishow',
2208                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2209                 'upload_date': '20140324',
2210                 'uploader': 'SciShow',
2211                 'like_count': int,
2212                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2213                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2214                 'view_count': int,
2215                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2216                 'playable_in_embed': True,
2217                 'tags': 'count:12',
2218                 'uploader_url': 'http://www.youtube.com/user/scishow',
2219                 'availability': 'public',
2220                 'channel': 'SciShow',
2221                 'live_status': 'not_live',
2222                 'duration': 248,
2223                 'categories': ['Education'],
2224                 'age_limit': 0,
2225                 'channel_follower_count': int,
2226                 'chapters': list,
2227             }, 'params': {'format': 'mhtml', 'skip_download': True}
2228         }, {
2229             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2230             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2231             'info_dict': {
2232                 'id': '2NUZ8W2llS4',
2233                 'ext': 'mp4',
2234                 'title': 'The NP that test your phone performance 🙂',
2235                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2236                 'uploader': 'Leon Nguyen',
2237                 'uploader_id': 'VNSXIII',
2238                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2239                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2240                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2241                 'duration': 21,
2242                 'view_count': int,
2243                 'age_limit': 0,
2244                 'categories': ['Gaming'],
2245                 'tags': 'count:23',
2246                 'playable_in_embed': True,
2247                 'live_status': 'not_live',
2248                 'upload_date': '20220103',
2249                 'like_count': int,
2250                 'availability': 'public',
2251                 'channel': 'Leon Nguyen',
2252                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2253                 'comment_count': int,
2254                 'channel_follower_count': int
2255             }
2256         }, {
2257             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2258             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2259             'info_dict': {
2260                 'id': '2NUZ8W2llS4',
2261                 'ext': 'mp4',
2262                 'title': 'The NP that test your phone performance 🙂',
2263                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2264                 'uploader': 'Leon Nguyen',
2265                 'uploader_id': 'VNSXIII',
2266                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2267                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2268                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2269                 'duration': 21,
2270                 'view_count': int,
2271                 'age_limit': 0,
2272                 'categories': ['Gaming'],
2273                 'tags': 'count:23',
2274                 'playable_in_embed': True,
2275                 'live_status': 'not_live',
2276                 'upload_date': '20220102',
2277                 'like_count': int,
2278                 'availability': 'public',
2279                 'channel': 'Leon Nguyen',
2280                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2281                 'comment_count': int,
2282                 'channel_follower_count': int
2283             },
2284             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2285         }, {
2286             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2287             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2288             'info_dict': {
2289                 'id': 'mzZzzBU6lrM',
2290                 'ext': 'mp4',
2291                 'title': 'I Met GeorgeNotFound In Real Life...',
2292                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2293                 'uploader': 'Quackity',
2294                 'uploader_id': 'QuackityHQ',
2295                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2296                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2297                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2298                 'duration': 955,
2299                 'view_count': int,
2300                 'age_limit': 0,
2301                 'categories': ['Entertainment'],
2302                 'tags': 'count:26',
2303                 'playable_in_embed': True,
2304                 'live_status': 'not_live',
2305                 'release_timestamp': 1641172509,
2306                 'release_date': '20220103',
2307                 'upload_date': '20220103',
2308                 'like_count': int,
2309                 'availability': 'public',
2310                 'channel': 'Quackity',
2311                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2312                 'channel_follower_count': int
2313             }
2314         },
2315         {   # continuous livestream. Microformat upload date should be preferred.
2316             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2317             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2318             'info_dict': {
2319                 'id': 'kgx4WGK0oNU',
2320                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2321                 'ext': 'mp4',
2322                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2323                 'availability': 'public',
2324                 'age_limit': 0,
2325                 'release_timestamp': 1637975704,
2326                 'upload_date': '20210619',
2327                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2328                 'live_status': 'is_live',
2329                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2330                 'uploader': '阿鲍Abao',
2331                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2332                 'channel': 'Abao in Tokyo',
2333                 'channel_follower_count': int,
2334                 'release_date': '20211127',
2335                 'tags': 'count:39',
2336                 'categories': ['People & Blogs'],
2337                 'like_count': int,
2338                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2339                 'view_count': int,
2340                 'playable_in_embed': True,
2341                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2342                 'concurrent_view_count': int,
2343             },
2344             'params': {'skip_download': True}
2345         }, {
2346             # Story. Requires specific player params to work.
2347             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2348             'info_dict': {
2349                 'id': 'vv8qTUWmulI',
2350                 'ext': 'mp4',
2351                 'availability': 'unlisted',
2352                 'view_count': int,
2353                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2354                 'upload_date': '20220526',
2355                 'categories': ['Education'],
2356                 'title': 'Story',
2357                 'channel': 'IT\'S HISTORY',
2358                 'description': '',
2359                 'uploader_id': 'BlastfromthePast',
2360                 'duration': 12,
2361                 'uploader': 'IT\'S HISTORY',
2362                 'playable_in_embed': True,
2363                 'age_limit': 0,
2364                 'live_status': 'not_live',
2365                 'tags': [],
2366                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2367                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2368                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2369             },
2370             'skip': 'stories get removed after some period of time',
2371         }, {
2372             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2373             'info_dict': {
2374                 'id': 'tjjjtzRLHvA',
2375                 'ext': 'mp4',
2376                 'title': 'ハッシュタグ無し };if window.ytcsi',
2377                 'upload_date': '20220323',
2378                 'like_count': int,
2379                 'availability': 'unlisted',
2380                 'channel': 'nao20010128nao',
2381                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2382                 'age_limit': 0,
2383                 'uploader': 'nao20010128nao',
2384                 'uploader_id': 'nao20010128nao',
2385                 'categories': ['Music'],
2386                 'view_count': int,
2387                 'description': '',
2388                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2389                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2390                 'live_status': 'not_live',
2391                 'playable_in_embed': True,
2392                 'channel_follower_count': int,
2393                 'duration': 6,
2394                 'tags': [],
2395                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2396             }
2397         }, {
2398             # Prefer primary title+description language metadata by default
2399             # Do not prefer translated description if primary is empty
2400             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2401             'info_dict': {
2402                 'id': 'el3E4MbxRqQ',
2403                 'ext': 'mp4',
2404                 'title': 'dlp test video 2 - primary sv no desc',
2405                 'description': '',
2406                 'channel': 'cole-dlp-test-acc',
2407                 'tags': [],
2408                 'view_count': int,
2409                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2410                 'like_count': int,
2411                 'playable_in_embed': True,
2412                 'availability': 'unlisted',
2413                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2414                 'age_limit': 0,
2415                 'duration': 5,
2416                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2417                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2418                 'live_status': 'not_live',
2419                 'upload_date': '20220908',
2420                 'categories': ['People & Blogs'],
2421                 'uploader': 'cole-dlp-test-acc',
2422                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2423             },
2424             'params': {'skip_download': True}
2425         }, {
2426             # Extractor argument: prefer translated title+description
2427             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2428             'info_dict': {
2429                 'id': 'gHKT4uU8Zng',
2430                 'ext': 'mp4',
2431                 'channel': 'cole-dlp-test-acc',
2432                 'tags': [],
2433                 'duration': 5,
2434                 'live_status': 'not_live',
2435                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2436                 'upload_date': '20220728',
2437                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2438                 'view_count': int,
2439                 'categories': ['People & Blogs'],
2440                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2441                 'title': 'dlp test video title translated (fr)',
2442                 'availability': 'public',
2443                 'uploader': 'cole-dlp-test-acc',
2444                 'age_limit': 0,
2445                 'description': 'dlp test video description translated (fr)',
2446                 'playable_in_embed': True,
2447                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2448                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2449             },
2450             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2451             'expected_warnings': [r'Preferring "fr" translated fields'],
2452         }, {
2453             'note': '6 channel audio',
2454             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2455             'only_matching': True,
2456         }
2457     ]
2458
2459     _WEBPAGE_TESTS = [
2460         # YouTube <object> embed
2461         {
2462             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2463             'md5': '873c81d308b979f0e23ee7e620b312a3',
2464             'info_dict': {
2465                 'id': 'msN87y-iEx0',
2466                 'ext': 'mp4',
2467                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2468                 'upload_date': '20080526',
2469                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2470                 'uploader': 'Christopher Sykes',
2471                 'uploader_id': 'ChristopherJSykes',
2472                 'age_limit': 0,
2473                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2474                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2475                 'playable_in_embed': True,
2476                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2477                 'like_count': int,
2478                 'comment_count': int,
2479                 'channel': 'Christopher Sykes',
2480                 'live_status': 'not_live',
2481                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2482                 'availability': 'public',
2483                 'duration': 195,
2484                 'view_count': int,
2485                 'categories': ['Science & Technology'],
2486                 'channel_follower_count': int,
2487                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2488             },
2489             'params': {
2490                 'skip_download': True,
2491             }
2492         },
2493     ]
2494
2495     @classmethod
2496     def suitable(cls, url):
2497         from ..utils import parse_qs
2498
2499         qs = parse_qs(url)
2500         if qs.get('list', [None])[0]:
2501             return False
2502         return super().suitable(url)
2503
2504     def __init__(self, *args, **kwargs):
2505         super().__init__(*args, **kwargs)
2506         self._code_cache = {}
2507         self._player_cache = {}
2508
2509     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2510         lock = threading.Lock()
2511         start_time = time.time()
2512         formats = [f for f in formats if f.get('is_from_start')]
2513
2514         def refetch_manifest(format_id, delay):
2515             nonlocal formats, start_time, is_live
2516             if time.time() <= start_time + delay:
2517                 return
2518
2519             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2520             video_details = traverse_obj(
2521                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2522             microformats = traverse_obj(
2523                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2524                 expected_type=dict, default=[])
2525             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2526             is_live = live_status == 'is_live'
2527             start_time = time.time()
2528
2529         def mpd_feed(format_id, delay):
2530             """
2531             @returns (manifest_url, manifest_stream_number, is_live) or None
2532             """
2533             with lock:
2534                 refetch_manifest(format_id, delay)
2535
2536             f = next((f for f in formats if f['format_id'] == format_id), None)
2537             if not f:
2538                 if not is_live:
2539                     self.to_screen(f'{video_id}: Video is no longer live')
2540                 else:
2541                     self.report_warning(
2542                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2543                 return None
2544             return f['manifest_url'], f['manifest_stream_number'], is_live
2545
2546         for f in formats:
2547             f['is_live'] = is_live
2548             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2549                                     live_start_time, mpd_feed, not is_live and f.copy())
2550             if is_live:
2551                 f['fragments'] = gen
2552                 f['protocol'] = 'http_dash_segments_generator'
2553             else:
2554                 f['fragments'] = LazyList(gen({}))
2555                 del f['is_from_start']
2556
2557     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2558         FETCH_SPAN, MAX_DURATION = 5, 432000
2559
2560         mpd_url, stream_number, is_live = None, None, True
2561
2562         begin_index = 0
2563         download_start_time = ctx.get('start') or time.time()
2564
2565         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2566         if lack_early_segments:
2567             self.report_warning(bug_reports_message(
2568                 'Starting download from the last 120 hours of the live stream since '
2569                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2570             lack_early_segments = True
2571
2572         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2573         fragments, fragment_base_url = None, None
2574
2575         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2576             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2577             # Obtain from MPD's maximum seq value
2578             old_mpd_url = mpd_url
2579             last_error = ctx.pop('last_error', None)
2580             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2581             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2582                                                or (mpd_url, stream_number, False))
2583             if not refresh_sequence:
2584                 if expire_fast and not is_live:
2585                     return False, last_seq
2586                 elif old_mpd_url == mpd_url:
2587                     return True, last_seq
2588             if manifestless_orig_fmt:
2589                 fmt_info = manifestless_orig_fmt
2590             else:
2591                 try:
2592                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2593                         mpd_url, None, note=False, errnote=False, fatal=False)
2594                 except ExtractorError:
2595                     fmts = None
2596                 if not fmts:
2597                     no_fragment_score += 2
2598                     return False, last_seq
2599                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2600             fragments = fmt_info['fragments']
2601             fragment_base_url = fmt_info['fragment_base_url']
2602             assert fragment_base_url
2603
2604             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2605             return True, _last_seq
2606
2607         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2608         while is_live:
2609             fetch_time = time.time()
2610             if no_fragment_score > 30:
2611                 return
2612             if last_segment_url:
2613                 # Obtain from "X-Head-Seqnum" header value from each segment
2614                 try:
2615                     urlh = self._request_webpage(
2616                         last_segment_url, None, note=False, errnote=False, fatal=False)
2617                 except ExtractorError:
2618                     urlh = None
2619                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2620                 if last_seq is None:
2621                     no_fragment_score += 2
2622                     last_segment_url = None
2623                     continue
2624             else:
2625                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2626                 no_fragment_score += 2
2627                 if not should_continue:
2628                     continue
2629
2630             if known_idx > last_seq:
2631                 last_segment_url = None
2632                 continue
2633
2634             last_seq += 1
2635
2636             if begin_index < 0 and known_idx < 0:
2637                 # skip from the start when it's negative value
2638                 known_idx = last_seq + begin_index
2639             if lack_early_segments:
2640                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2641             try:
2642                 for idx in range(known_idx, last_seq):
2643                     # do not update sequence here or you'll get skipped some part of it
2644                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2645                     if not should_continue:
2646                         known_idx = idx - 1
2647                         raise ExtractorError('breaking out of outer loop')
2648                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2649                     yield {
2650                         'url': last_segment_url,
2651                         'fragment_count': last_seq,
2652                     }
2653                 if known_idx == last_seq:
2654                     no_fragment_score += 5
2655                 else:
2656                     no_fragment_score = 0
2657                 known_idx = last_seq
2658             except ExtractorError:
2659                 continue
2660
2661             if manifestless_orig_fmt:
2662                 # Stop at the first iteration if running for post-live manifestless;
2663                 # fragment count no longer increase since it starts
2664                 break
2665
2666             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2667
2668     def _extract_player_url(self, *ytcfgs, webpage=None):
2669         player_url = traverse_obj(
2670             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2671             get_all=False, expected_type=str)
2672         if not player_url:
2673             return
2674         return urljoin('https://www.youtube.com', player_url)
2675
2676     def _download_player_url(self, video_id, fatal=False):
2677         res = self._download_webpage(
2678             'https://www.youtube.com/iframe_api',
2679             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2680         if res:
2681             player_version = self._search_regex(
2682                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2683             if player_version:
2684                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2685
2686     def _signature_cache_id(self, example_sig):
2687         """ Return a string representation of a signature """
2688         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2689
2690     @classmethod
2691     def _extract_player_info(cls, player_url):
2692         for player_re in cls._PLAYER_INFO_RE:
2693             id_m = re.search(player_re, player_url)
2694             if id_m:
2695                 break
2696         else:
2697             raise ExtractorError('Cannot identify player %r' % player_url)
2698         return id_m.group('id')
2699
2700     def _load_player(self, video_id, player_url, fatal=True):
2701         player_id = self._extract_player_info(player_url)
2702         if player_id not in self._code_cache:
2703             code = self._download_webpage(
2704                 player_url, video_id, fatal=fatal,
2705                 note='Downloading player ' + player_id,
2706                 errnote='Download of %s failed' % player_url)
2707             if code:
2708                 self._code_cache[player_id] = code
2709         return self._code_cache.get(player_id)
2710
2711     def _extract_signature_function(self, video_id, player_url, example_sig):
2712         player_id = self._extract_player_info(player_url)
2713
2714         # Read from filesystem cache
2715         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2716         assert os.path.basename(func_id) == func_id
2717
2718         self.write_debug(f'Extracting signature function {func_id}')
2719         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2720
2721         if not cache_spec:
2722             code = self._load_player(video_id, player_url)
2723         if code:
2724             res = self._parse_sig_js(code)
2725             test_string = ''.join(map(chr, range(len(example_sig))))
2726             cache_spec = [ord(c) for c in res(test_string)]
2727             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2728
2729         return lambda s: ''.join(s[i] for i in cache_spec)
2730
2731     def _print_sig_code(self, func, example_sig):
2732         if not self.get_param('youtube_print_sig_code'):
2733             return
2734
2735         def gen_sig_code(idxs):
2736             def _genslice(start, end, step):
2737                 starts = '' if start == 0 else str(start)
2738                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2739                 steps = '' if step == 1 else (':%d' % step)
2740                 return f's[{starts}{ends}{steps}]'
2741
2742             step = None
2743             # Quelch pyflakes warnings - start will be set when step is set
2744             start = '(Never used)'
2745             for i, prev in zip(idxs[1:], idxs[:-1]):
2746                 if step is not None:
2747                     if i - prev == step:
2748                         continue
2749                     yield _genslice(start, prev, step)
2750                     step = None
2751                     continue
2752                 if i - prev in [-1, 1]:
2753                     step = i - prev
2754                     start = prev
2755                     continue
2756                 else:
2757                     yield 's[%d]' % prev
2758             if step is None:
2759                 yield 's[%d]' % i
2760             else:
2761                 yield _genslice(start, i, step)
2762
2763         test_string = ''.join(map(chr, range(len(example_sig))))
2764         cache_res = func(test_string)
2765         cache_spec = [ord(c) for c in cache_res]
2766         expr_code = ' + '.join(gen_sig_code(cache_spec))
2767         signature_id_tuple = '(%s)' % (
2768             ', '.join(str(len(p)) for p in example_sig.split('.')))
2769         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2770                 '    return %s\n') % (signature_id_tuple, expr_code)
2771         self.to_screen('Extracted signature function:\n' + code)
2772
2773     def _parse_sig_js(self, jscode):
2774         funcname = self._search_regex(
2775             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2776              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2777              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2778              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2779              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2780              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2781              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2782              # Obsolete patterns
2783              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2784              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2785              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2786              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2787              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2788              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2789              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2790              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2791             jscode, 'Initial JS player signature function name', group='sig')
2792
2793         jsi = JSInterpreter(jscode)
2794         initial_function = jsi.extract_function(funcname)
2795         return lambda s: initial_function([s])
2796
2797     def _cached(self, func, *cache_id):
2798         def inner(*args, **kwargs):
2799             if cache_id not in self._player_cache:
2800                 try:
2801                     self._player_cache[cache_id] = func(*args, **kwargs)
2802                 except ExtractorError as e:
2803                     self._player_cache[cache_id] = e
2804                 except Exception as e:
2805                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
2806
2807             ret = self._player_cache[cache_id]
2808             if isinstance(ret, Exception):
2809                 raise ret
2810             return ret
2811         return inner
2812
2813     def _decrypt_signature(self, s, video_id, player_url):
2814         """Turn the encrypted s field into a working signature"""
2815         extract_sig = self._cached(
2816             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
2817         func = extract_sig(video_id, player_url, s)
2818         self._print_sig_code(func, s)
2819         return func(s)
2820
2821     def _decrypt_nsig(self, s, video_id, player_url):
2822         """Turn the encrypted n field into a working signature"""
2823         if player_url is None:
2824             raise ExtractorError('Cannot decrypt nsig without player_url')
2825         player_url = urljoin('https://www.youtube.com', player_url)
2826
2827         try:
2828             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
2829         except ExtractorError as e:
2830             raise ExtractorError('Unable to extract nsig function code', cause=e)
2831         if self.get_param('youtube_print_sig_code'):
2832             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2833
2834         try:
2835             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
2836             ret = extract_nsig(jsi, func_code)(s)
2837         except JSInterpreter.Exception as e:
2838             try:
2839                 jsi = PhantomJSwrapper(self, timeout=5000)
2840             except ExtractorError:
2841                 raise e
2842             self.report_warning(
2843                 f'Native nsig extraction failed: Trying with PhantomJS\n'
2844                 f'         n = {s} ; player = {player_url}', video_id)
2845             self.write_debug(e, only_once=True)
2846
2847             args, func_body = func_code
2848             ret = jsi.execute(
2849                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
2850                 video_id=video_id, note='Executing signature code').strip()
2851
2852         self.write_debug(f'Decrypted nsig {s} => {ret}')
2853         return ret
2854
2855     def _extract_n_function_name(self, jscode):
2856         funcname, idx = self._search_regex(
2857             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2858             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2859         if not idx:
2860             return funcname
2861
2862         return json.loads(js_to_json(self._search_regex(
2863             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
2864             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
2865
2866     def _extract_n_function_code(self, video_id, player_url):
2867         player_id = self._extract_player_info(player_url)
2868         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
2869         jscode = func_code or self._load_player(video_id, player_url)
2870         jsi = JSInterpreter(jscode)
2871
2872         if func_code:
2873             return jsi, player_id, func_code
2874
2875         func_name = self._extract_n_function_name(jscode)
2876
2877         # For redundancy
2878         func_code = self._search_regex(
2879             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
2880                      # NB: The end of the regex is intentionally kept strict
2881                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
2882             jscode, 'nsig function', group=('var', 'code'), default=None)
2883         if func_code:
2884             func_code = ([func_code[0]], func_code[1])
2885         else:
2886             self.write_debug('Extracting nsig function with jsinterp')
2887             func_code = jsi.extract_function_code(func_name)
2888
2889         self.cache.store('youtube-nsig', player_id, func_code)
2890         return jsi, player_id, func_code
2891
2892     def _extract_n_function_from_code(self, jsi, func_code):
2893         func = jsi.extract_function_from_code(*func_code)
2894
2895         def extract_nsig(s):
2896             try:
2897                 ret = func([s])
2898             except JSInterpreter.Exception:
2899                 raise
2900             except Exception as e:
2901                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
2902
2903             if ret.startswith('enhanced_except_'):
2904                 raise JSInterpreter.Exception('Signature function returned an exception')
2905             return ret
2906
2907         return extract_nsig
2908
2909     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2910         """
2911         Extract signatureTimestamp (sts)
2912         Required to tell API what sig/player version is in use.
2913         """
2914         sts = None
2915         if isinstance(ytcfg, dict):
2916             sts = int_or_none(ytcfg.get('STS'))
2917
2918         if not sts:
2919             # Attempt to extract from player
2920             if player_url is None:
2921                 error_msg = 'Cannot extract signature timestamp without player_url.'
2922                 if fatal:
2923                     raise ExtractorError(error_msg)
2924                 self.report_warning(error_msg)
2925                 return
2926             code = self._load_player(video_id, player_url, fatal=fatal)
2927             if code:
2928                 sts = int_or_none(self._search_regex(
2929                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2930                     'JS player signature timestamp', group='sts', fatal=fatal))
2931         return sts
2932
2933     def _mark_watched(self, video_id, player_responses):
2934         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2935             label = 'fully ' if is_full else ''
2936             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2937                             expected_type=url_or_none)
2938             if not url:
2939                 self.report_warning(f'Unable to mark {label}watched')
2940                 return
2941             parsed_url = urllib.parse.urlparse(url)
2942             qs = urllib.parse.parse_qs(parsed_url.query)
2943
2944             # cpn generation algorithm is reverse engineered from base.js.
2945             # In fact it works even with dummy cpn.
2946             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2947             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2948
2949             # # more consistent results setting it to right before the end
2950             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2951
2952             qs.update({
2953                 'ver': ['2'],
2954                 'cpn': [cpn],
2955                 'cmt': video_length,
2956                 'el': 'detailpage',  # otherwise defaults to "shorts"
2957             })
2958
2959             if is_full:
2960                 # these seem to mark watchtime "history" in the real world
2961                 # they're required, so send in a single value
2962                 qs.update({
2963                     'st': 0,
2964                     'et': video_length,
2965                 })
2966
2967             url = urllib.parse.urlunparse(
2968                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2969
2970             self._download_webpage(
2971                 url, video_id, f'Marking {label}watched',
2972                 'Unable to mark watched', fatal=False)
2973
2974     @classmethod
2975     def _extract_from_webpage(cls, url, webpage):
2976         # Invidious Instances
2977         # https://github.com/yt-dlp/yt-dlp/issues/195
2978         # https://github.com/iv-org/invidious/pull/1730
2979         mobj = re.search(
2980             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
2981             webpage)
2982         if mobj:
2983             yield cls.url_result(mobj.group('url'), cls)
2984             raise cls.StopExtraction()
2985
2986         yield from super()._extract_from_webpage(url, webpage)
2987
2988         # lazyYT YouTube embed
2989         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
2990             yield cls.url_result(unescapeHTML(id_), cls, id_)
2991
2992         # Wordpress "YouTube Video Importer" plugin
2993         for m in re.findall(r'''(?x)<div[^>]+
2994                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2995                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
2996             yield cls.url_result(m[-1], cls, m[-1])
2997
2998     @classmethod
2999     def extract_id(cls, url):
3000         video_id = cls.get_temp_id(url)
3001         if not video_id:
3002             raise ExtractorError(f'Invalid URL: {url}')
3003         return video_id
3004
3005     def _extract_chapters_from_json(self, data, duration):
3006         chapter_list = traverse_obj(
3007             data, (
3008                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3009                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3010             ), expected_type=list)
3011
3012         return self._extract_chapters(
3013             chapter_list,
3014             chapter_time=lambda chapter: float_or_none(
3015                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3016             chapter_title=lambda chapter: traverse_obj(
3017                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3018             duration=duration)
3019
3020     def _extract_chapters_from_engagement_panel(self, data, duration):
3021         content_list = traverse_obj(
3022             data,
3023             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3024             expected_type=list, default=[])
3025         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3026         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3027
3028         return next(filter(None, (
3029             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3030                                    chapter_time, chapter_title, duration)
3031             for contents in content_list)), [])
3032
3033     def _extract_chapters_from_description(self, description, duration):
3034         duration_re = r'(?:\d+:)?\d{1,2}:\d{2}'
3035         sep_re = r'(?m)^\s*(%s)\b\W*\s(%s)\s*$'
3036         return self._extract_chapters(
3037             re.findall(sep_re % (duration_re, r'.+?'), description or ''),
3038             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
3039             duration=duration, strict=False) or self._extract_chapters(
3040             re.findall(sep_re % (r'.+?', duration_re), description or ''),
3041             chapter_time=lambda x: parse_duration(x[1]), chapter_title=lambda x: x[0],
3042             duration=duration, strict=False)
3043
3044     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
3045         if not duration:
3046             return
3047         chapter_list = [{
3048             'start_time': chapter_time(chapter),
3049             'title': chapter_title(chapter),
3050         } for chapter in chapter_list or []]
3051         if not strict:
3052             chapter_list.sort(key=lambda c: c['start_time'] or 0)
3053
3054         chapters = [{'start_time': 0}]
3055         for idx, chapter in enumerate(chapter_list):
3056             if chapter['start_time'] is None:
3057                 self.report_warning(f'Incomplete chapter {idx}')
3058             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
3059                 chapters.append(chapter)
3060             elif chapter not in chapters:
3061                 self.report_warning(
3062                     f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
3063         return chapters[1:]
3064
3065     def _extract_comment(self, comment_renderer, parent=None):
3066         comment_id = comment_renderer.get('commentId')
3067         if not comment_id:
3068             return
3069
3070         text = self._get_text(comment_renderer, 'contentText')
3071
3072         # Timestamp is an estimate calculated from the current time and time_text
3073         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3074         timestamp = self._parse_time_text(time_text)
3075
3076         author = self._get_text(comment_renderer, 'authorText')
3077         author_id = try_get(comment_renderer,
3078                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3079
3080         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3081                                                        lambda x: x['likeCount']), str)) or 0
3082         author_thumbnail = try_get(comment_renderer,
3083                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3084
3085         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3086         is_favorited = 'creatorHeart' in (try_get(
3087             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3088         return {
3089             'id': comment_id,
3090             'text': text,
3091             'timestamp': timestamp,
3092             'time_text': time_text,
3093             'like_count': votes,
3094             'is_favorited': is_favorited,
3095             'author': author,
3096             'author_id': author_id,
3097             'author_thumbnail': author_thumbnail,
3098             'author_is_uploader': author_is_uploader,
3099             'parent': parent or 'root'
3100         }
3101
3102     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3103
3104         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3105
3106         def extract_header(contents):
3107             _continuation = None
3108             for content in contents:
3109                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3110                 expected_comment_count = self._get_count(
3111                     comments_header_renderer, 'countText', 'commentsCount')
3112
3113                 if expected_comment_count:
3114                     tracker['est_total'] = expected_comment_count
3115                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3116                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3117
3118                 sort_menu_item = try_get(
3119                     comments_header_renderer,
3120                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3121                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3122
3123                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3124                 if not _continuation:
3125                     continue
3126
3127                 sort_text = str_or_none(sort_menu_item.get('title'))
3128                 if not sort_text:
3129                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3130                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3131                 break
3132             return _continuation
3133
3134         def extract_thread(contents):
3135             if not parent:
3136                 tracker['current_page_thread'] = 0
3137             for content in contents:
3138                 if not parent and tracker['total_parent_comments'] >= max_parents:
3139                     yield
3140                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3141                 comment_renderer = get_first(
3142                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3143                     expected_type=dict, default={})
3144
3145                 comment = self._extract_comment(comment_renderer, parent)
3146                 if not comment:
3147                     continue
3148
3149                 tracker['running_total'] += 1
3150                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3151                 yield comment
3152
3153                 # Attempt to get the replies
3154                 comment_replies_renderer = try_get(
3155                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3156
3157                 if comment_replies_renderer:
3158                     tracker['current_page_thread'] += 1
3159                     comment_entries_iter = self._comment_entries(
3160                         comment_replies_renderer, ytcfg, video_id,
3161                         parent=comment.get('id'), tracker=tracker)
3162                     yield from itertools.islice(comment_entries_iter, min(
3163                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3164
3165         # Keeps track of counts across recursive calls
3166         if not tracker:
3167             tracker = dict(
3168                 running_total=0,
3169                 est_total=0,
3170                 current_page_thread=0,
3171                 total_parent_comments=0,
3172                 total_reply_comments=0)
3173
3174         # TODO: Deprecated
3175         # YouTube comments have a max depth of 2
3176         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3177         if max_depth:
3178             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3179                                                 'Set max replies in the max-comments extractor argument instead')
3180         if max_depth == 1 and parent:
3181             return
3182
3183         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3184             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3185
3186         continuation = self._extract_continuation(root_continuation_data)
3187
3188         response = None
3189         is_forced_continuation = False
3190         is_first_continuation = parent is None
3191         if is_first_continuation and not continuation:
3192             # Sometimes you can get comments by generating the continuation yourself,
3193             # even if YouTube initially reports them being disabled - e.g. stories comments.
3194             # Note: if the comment section is actually disabled, YouTube may return a response with
3195             # required check_get_keys missing. So we will disable that check initially in this case.
3196             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3197             is_forced_continuation = True
3198
3199         for page_num in itertools.count(0):
3200             if not continuation:
3201                 break
3202             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3203             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3204             if page_num == 0:
3205                 if is_first_continuation:
3206                     note_prefix = 'Downloading comment section API JSON'
3207                 else:
3208                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3209                         tracker['current_page_thread'], comment_prog_str)
3210             else:
3211                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3212                     '       ' if parent else '', ' replies' if parent else '',
3213                     page_num, comment_prog_str)
3214
3215             response = self._extract_response(
3216                 item_id=None, query=continuation,
3217                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3218                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3219             is_forced_continuation = False
3220             continuation_contents = traverse_obj(
3221                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3222
3223             continuation = None
3224             for continuation_section in continuation_contents:
3225                 continuation_items = traverse_obj(
3226                     continuation_section,
3227                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3228                     get_all=False, expected_type=list) or []
3229                 if is_first_continuation:
3230                     continuation = extract_header(continuation_items)
3231                     is_first_continuation = False
3232                     if continuation:
3233                         break
3234                     continue
3235
3236                 for entry in extract_thread(continuation_items):
3237                     if not entry:
3238                         return
3239                     yield entry
3240                 continuation = self._extract_continuation({'contents': continuation_items})
3241                 if continuation:
3242                     break
3243
3244         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3245         if message and not parent and tracker['running_total'] == 0:
3246             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3247
3248     @staticmethod
3249     def _generate_comment_continuation(video_id):
3250         """
3251         Generates initial comment section continuation token from given video id
3252         """
3253         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3254         return base64.b64encode(token.encode()).decode()
3255
3256     def _get_comments(self, ytcfg, video_id, contents, webpage):
3257         """Entry for comment extraction"""
3258         def _real_comment_extract(contents):
3259             renderer = next((
3260                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3261                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3262             yield from self._comment_entries(renderer, ytcfg, video_id)
3263
3264         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3265         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3266
3267     @staticmethod
3268     def _get_checkok_params():
3269         return {'contentCheckOk': True, 'racyCheckOk': True}
3270
3271     @classmethod
3272     def _generate_player_context(cls, sts=None):
3273         context = {
3274             'html5Preference': 'HTML5_PREF_WANTS',
3275         }
3276         if sts is not None:
3277             context['signatureTimestamp'] = sts
3278         return {
3279             'playbackContext': {
3280                 'contentPlaybackContext': context
3281             },
3282             **cls._get_checkok_params()
3283         }
3284
3285     @staticmethod
3286     def _is_agegated(player_response):
3287         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3288             return True
3289
3290         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3291         AGE_GATE_REASONS = (
3292             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3293             'age_verification_required', 'age_check_required',  # status
3294         )
3295         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3296
3297     @staticmethod
3298     def _is_unplayable(player_response):
3299         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3300
3301     _STORY_PLAYER_PARAMS = '8AEB'
3302
3303     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3304
3305         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3306         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3307         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3308         headers = self.generate_api_headers(
3309             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3310
3311         yt_query = {
3312             'videoId': video_id,
3313         }
3314         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3315             yt_query['params'] = self._STORY_PLAYER_PARAMS
3316
3317         yt_query.update(self._generate_player_context(sts))
3318         return self._extract_response(
3319             item_id=video_id, ep='player', query=yt_query,
3320             ytcfg=player_ytcfg, headers=headers, fatal=True,
3321             default_client=client,
3322             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3323         ) or None
3324
3325     def _get_requested_clients(self, url, smuggled_data):
3326         requested_clients = []
3327         default = ['android', 'web']
3328         allowed_clients = sorted(
3329             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3330             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3331         for client in self._configuration_arg('player_client'):
3332             if client in allowed_clients:
3333                 requested_clients.append(client)
3334             elif client == 'default':
3335                 requested_clients.extend(default)
3336             elif client == 'all':
3337                 requested_clients.extend(allowed_clients)
3338             else:
3339                 self.report_warning(f'Skipping unsupported client {client}')
3340         if not requested_clients:
3341             requested_clients = default
3342
3343         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3344             requested_clients.extend(
3345                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3346
3347         return orderedSet(requested_clients)
3348
3349     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3350         initial_pr = None
3351         if webpage:
3352             initial_pr = self._search_json(
3353                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3354
3355         all_clients = set(clients)
3356         clients = clients[::-1]
3357         prs = []
3358
3359         def append_client(*client_names):
3360             """ Append the first client name that exists but not already used """
3361             for client_name in client_names:
3362                 actual_client = _split_innertube_client(client_name)[0]
3363                 if actual_client in INNERTUBE_CLIENTS:
3364                     if actual_client not in all_clients:
3365                         clients.append(client_name)
3366                         all_clients.add(actual_client)
3367                         return
3368
3369         # Android player_response does not have microFormats which are needed for
3370         # extraction of some data. So we return the initial_pr with formats
3371         # stripped out even if not requested by the user
3372         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3373         if initial_pr:
3374             pr = dict(initial_pr)
3375             pr['streamingData'] = None
3376             prs.append(pr)
3377
3378         last_error = None
3379         tried_iframe_fallback = False
3380         player_url = None
3381         while clients:
3382             client, base_client, variant = _split_innertube_client(clients.pop())
3383             player_ytcfg = master_ytcfg if client == 'web' else {}
3384             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3385                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3386
3387             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3388             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3389             if 'js' in self._configuration_arg('player_skip'):
3390                 require_js_player = False
3391                 player_url = None
3392
3393             if not player_url and not tried_iframe_fallback and require_js_player:
3394                 player_url = self._download_player_url(video_id)
3395                 tried_iframe_fallback = True
3396
3397             try:
3398                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3399                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3400             except ExtractorError as e:
3401                 if last_error:
3402                     self.report_warning(last_error)
3403                 last_error = e
3404                 continue
3405
3406             if pr:
3407                 # YouTube may return a different video player response than expected.
3408                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3409                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3410                 if pr_video_id and pr_video_id != video_id:
3411                     self.report_warning(
3412                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3413                 else:
3414                     prs.append(pr)
3415
3416             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3417             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3418                 append_client(f'{base_client}_creator')
3419             elif self._is_agegated(pr):
3420                 if variant == 'tv_embedded':
3421                     append_client(f'{base_client}_embedded')
3422                 elif not variant:
3423                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3424
3425         if last_error:
3426             if not len(prs):
3427                 raise last_error
3428             self.report_warning(last_error)
3429         return prs, player_url
3430
3431     def _needs_live_processing(self, live_status, duration):
3432         if (live_status == 'is_live' and self.get_param('live_from_start')
3433                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3434             return live_status
3435
3436     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3437         itags, stream_ids = {}, []
3438         itag_qualities, res_qualities = {}, {0: None}
3439         q = qualities([
3440             # Normally tiny is the smallest video-only formats. But
3441             # audio-only formats with unknown quality may get tagged as tiny
3442             'tiny',
3443             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3444             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3445         ])
3446         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3447
3448         for fmt in streaming_formats:
3449             if fmt.get('targetDurationSec'):
3450                 continue
3451
3452             itag = str_or_none(fmt.get('itag'))
3453             audio_track = fmt.get('audioTrack') or {}
3454             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3455             if stream_id in stream_ids:
3456                 continue
3457
3458             quality = fmt.get('quality')
3459             height = int_or_none(fmt.get('height'))
3460             if quality == 'tiny' or not quality:
3461                 quality = fmt.get('audioQuality', '').lower() or quality
3462             # The 3gp format (17) in android client has a quality of "small",
3463             # but is actually worse than other formats
3464             if itag == '17':
3465                 quality = 'tiny'
3466             if quality:
3467                 if itag:
3468                     itag_qualities[itag] = quality
3469                 if height:
3470                     res_qualities[height] = quality
3471             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3472             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3473             # number of fragment that would subsequently requested with (`&sq=N`)
3474             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3475                 continue
3476
3477             fmt_url = fmt.get('url')
3478             if not fmt_url:
3479                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3480                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3481                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3482                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3483                     continue
3484                 try:
3485                     fmt_url += '&%s=%s' % (
3486                         traverse_obj(sc, ('sp', -1)) or 'signature',
3487                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3488                     )
3489                 except ExtractorError as e:
3490                     self.report_warning('Signature extraction failed: Some formats may be missing',
3491                                         video_id=video_id, only_once=True)
3492                     self.write_debug(e, only_once=True)
3493                     continue
3494
3495             query = parse_qs(fmt_url)
3496             throttled = False
3497             if query.get('n'):
3498                 try:
3499                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3500                     fmt_url = update_url_query(fmt_url, {
3501                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3502                     })
3503                 except ExtractorError as e:
3504                     phantomjs_hint = ''
3505                     if isinstance(e, JSInterpreter.Exception):
3506                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3507                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3508                     if player_url:
3509                         self.report_warning(
3510                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3511                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3512                         self.write_debug(e, only_once=True)
3513                     else:
3514                         self.report_warning(
3515                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3516                             video_id=video_id, only_once=True)
3517                     throttled = True
3518
3519             if itag:
3520                 itags[itag] = 'https'
3521                 stream_ids.append(stream_id)
3522
3523             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3524             language_preference = (
3525                 10 if audio_track.get('audioIsDefault') and 10
3526                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3527                 else -1)
3528             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3529             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3530             # Make sure to avoid false positives with small duration differences.
3531             # E.g. __2ABJjxzNo, ySuUZEjARPY
3532             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3533             if is_damaged:
3534                 self.report_warning(
3535                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3536             dct = {
3537                 'asr': int_or_none(fmt.get('audioSampleRate')),
3538                 'filesize': int_or_none(fmt.get('contentLength')),
3539                 'format_id': itag,
3540                 'format_note': join_nonempty(
3541                     '%s%s' % (audio_track.get('displayName') or '',
3542                               ' (default)' if language_preference > 0 else ''),
3543                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3544                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3545                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3546                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3547                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3548                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3549                 'fps': int_or_none(fmt.get('fps')) or None,
3550                 'audio_channels': fmt.get('audioChannels'),
3551                 'height': height,
3552                 'quality': q(quality),
3553                 'has_drm': bool(fmt.get('drmFamilies')),
3554                 'tbr': tbr,
3555                 'url': fmt_url,
3556                 'width': int_or_none(fmt.get('width')),
3557                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3558                                           'desc' if language_preference < -1 else ''),
3559                 'language_preference': language_preference,
3560                 # Strictly de-prioritize damaged and 3gp formats
3561                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3562             }
3563             mime_mobj = re.match(
3564                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3565             if mime_mobj:
3566                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3567                 dct.update(parse_codecs(mime_mobj.group(2)))
3568             no_audio = dct.get('acodec') == 'none'
3569             no_video = dct.get('vcodec') == 'none'
3570             if no_audio:
3571                 dct['vbr'] = tbr
3572             if no_video:
3573                 dct['abr'] = tbr
3574             if no_audio or no_video:
3575                 dct['downloader_options'] = {
3576                     # Youtube throttles chunks >~10M
3577                     'http_chunk_size': 10485760,
3578                 }
3579                 if dct.get('ext'):
3580                     dct['container'] = dct['ext'] + '_dash'
3581             yield dct
3582
3583         needs_live_processing = self._needs_live_processing(live_status, duration)
3584         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3585
3586         skip_manifests = set(self._configuration_arg('skip'))
3587         if (not self.get_param('youtube_include_hls_manifest', True)
3588                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3589                 or needs_live_processing and skip_bad_formats):
3590             skip_manifests.add('hls')
3591
3592         if not self.get_param('youtube_include_dash_manifest', True):
3593             skip_manifests.add('dash')
3594         if self._configuration_arg('include_live_dash'):
3595             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3596                                                 'Use include_incomplete_formats extractor argument instead')
3597         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3598             skip_manifests.add('dash')
3599
3600         def process_manifest_format(f, proto, itag):
3601             if itag in itags:
3602                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3603                     return False
3604                 itag = f'{itag}-{proto}'
3605             if itag:
3606                 f['format_id'] = itag
3607                 itags[itag] = proto
3608
3609             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3610             if f['quality'] == -1 and f.get('height'):
3611                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3612             return True
3613
3614         subtitles = {}
3615         for sd in streaming_data:
3616             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3617             if hls_manifest_url:
3618                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3619                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3620                 subtitles = self._merge_subtitles(subs, subtitles)
3621                 for f in fmts:
3622                     if process_manifest_format(f, 'hls', self._search_regex(
3623                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3624                         yield f
3625
3626             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3627             if dash_manifest_url:
3628                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3629                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3630                 for f in formats:
3631                     if process_manifest_format(f, 'dash', f['format_id']):
3632                         f['filesize'] = int_or_none(self._search_regex(
3633                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3634                         if needs_live_processing:
3635                             f['is_from_start'] = True
3636
3637                         yield f
3638         yield subtitles
3639
3640     def _extract_storyboard(self, player_responses, duration):
3641         spec = get_first(
3642             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3643         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3644         if not base_url:
3645             return
3646         L = len(spec) - 1
3647         for i, args in enumerate(spec):
3648             args = args.split('#')
3649             counts = list(map(int_or_none, args[:5]))
3650             if len(args) != 8 or not all(counts):
3651                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3652                 continue
3653             width, height, frame_count, cols, rows = counts
3654             N, sigh = args[6:]
3655
3656             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3657             fragment_count = frame_count / (cols * rows)
3658             fragment_duration = duration / fragment_count
3659             yield {
3660                 'format_id': f'sb{i}',
3661                 'format_note': 'storyboard',
3662                 'ext': 'mhtml',
3663                 'protocol': 'mhtml',
3664                 'acodec': 'none',
3665                 'vcodec': 'none',
3666                 'url': url,
3667                 'width': width,
3668                 'height': height,
3669                 'fps': frame_count / duration,
3670                 'rows': rows,
3671                 'columns': cols,
3672                 'fragments': [{
3673                     'url': url.replace('$M', str(j)),
3674                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3675                 } for j in range(math.ceil(fragment_count))],
3676             }
3677
3678     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3679         webpage = None
3680         if 'webpage' not in self._configuration_arg('player_skip'):
3681             query = {'bpctr': '9999999999', 'has_verified': '1'}
3682             if smuggled_data.get('is_story'):
3683                 query['pp'] = self._STORY_PLAYER_PARAMS
3684             webpage = self._download_webpage(
3685                 webpage_url, video_id, fatal=False, query=query)
3686
3687         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3688
3689         player_responses, player_url = self._extract_player_responses(
3690             self._get_requested_clients(url, smuggled_data),
3691             video_id, webpage, master_ytcfg, smuggled_data)
3692
3693         return webpage, master_ytcfg, player_responses, player_url
3694
3695     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3696         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3697         is_live = get_first(video_details, 'isLive')
3698         if is_live is None:
3699             is_live = get_first(live_broadcast_details, 'isLiveNow')
3700         live_content = get_first(video_details, 'isLiveContent')
3701         is_upcoming = get_first(video_details, 'isUpcoming')
3702         post_live = get_first(video_details, 'isPostLiveDvr')
3703         live_status = ('post_live' if post_live
3704                        else 'is_live' if is_live
3705                        else 'is_upcoming' if is_upcoming
3706                        else 'was_live' if live_content
3707                        else 'not_live' if False in (is_live, live_content)
3708                        else None)
3709         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3710         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3711
3712         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3713
3714     def _real_extract(self, url):
3715         url, smuggled_data = unsmuggle_url(url, {})
3716         video_id = self._match_id(url)
3717
3718         base_url = self.http_scheme() + '//www.youtube.com/'
3719         webpage_url = base_url + 'watch?v=' + video_id
3720
3721         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3722
3723         playability_statuses = traverse_obj(
3724             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3725
3726         trailer_video_id = get_first(
3727             playability_statuses,
3728             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3729             expected_type=str)
3730         if trailer_video_id:
3731             return self.url_result(
3732                 trailer_video_id, self.ie_key(), trailer_video_id)
3733
3734         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3735                        if webpage else (lambda x: None))
3736
3737         video_details = traverse_obj(
3738             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3739         microformats = traverse_obj(
3740             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3741             expected_type=dict, default=[])
3742
3743         translated_title = self._get_text(microformats, (..., 'title'))
3744         video_title = (self._preferred_lang and translated_title
3745                        or get_first(video_details, 'title')  # primary
3746                        or translated_title
3747                        or search_meta(['og:title', 'twitter:title', 'title']))
3748         translated_description = self._get_text(microformats, (..., 'description'))
3749         original_description = get_first(video_details, 'shortDescription')
3750         video_description = (
3751             self._preferred_lang and translated_description
3752             # If original description is blank, it will be an empty string.
3753             # Do not prefer translated description in this case.
3754             or original_description if original_description is not None else translated_description)
3755
3756         multifeed_metadata_list = get_first(
3757             player_responses,
3758             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3759             expected_type=str)
3760         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3761             if self.get_param('noplaylist'):
3762                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3763             else:
3764                 entries = []
3765                 feed_ids = []
3766                 for feed in multifeed_metadata_list.split(','):
3767                     # Unquote should take place before split on comma (,) since textual
3768                     # fields may contain comma as well (see
3769                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3770                     feed_data = urllib.parse.parse_qs(
3771                         urllib.parse.unquote_plus(feed))
3772
3773                     def feed_entry(name):
3774                         return try_get(
3775                             feed_data, lambda x: x[name][0], str)
3776
3777                     feed_id = feed_entry('id')
3778                     if not feed_id:
3779                         continue
3780                     feed_title = feed_entry('title')
3781                     title = video_title
3782                     if feed_title:
3783                         title += ' (%s)' % feed_title
3784                     entries.append({
3785                         '_type': 'url_transparent',
3786                         'ie_key': 'Youtube',
3787                         'url': smuggle_url(
3788                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3789                             {'force_singlefeed': True}),
3790                         'title': title,
3791                     })
3792                     feed_ids.append(feed_id)
3793                 self.to_screen(
3794                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3795                     % (', '.join(feed_ids), video_id))
3796                 return self.playlist_result(
3797                     entries, video_id, video_title, video_description)
3798
3799         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
3800                     or int_or_none(get_first(microformats, 'lengthSeconds'))
3801                     or parse_duration(search_meta('duration')) or None)
3802
3803         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
3804             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
3805         if live_status == 'post_live':
3806             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
3807
3808         if not formats:
3809             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3810                 self.report_drm(video_id)
3811             pemr = get_first(
3812                 playability_statuses,
3813                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3814             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3815             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3816             if subreason:
3817                 if subreason == 'The uploader has not made this video available in your country.':
3818                     countries = get_first(microformats, 'availableCountries')
3819                     if not countries:
3820                         regions_allowed = search_meta('regionsAllowed')
3821                         countries = regions_allowed.split(',') if regions_allowed else None
3822                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3823                 reason += f'. {subreason}'
3824             if reason:
3825                 self.raise_no_formats(reason, expected=True)
3826
3827         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3828         if not keywords and webpage:
3829             keywords = [
3830                 unescapeHTML(m.group('content'))
3831                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3832         for keyword in keywords:
3833             if keyword.startswith('yt:stretch='):
3834                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3835                 if mobj:
3836                     # NB: float is intentional for forcing float division
3837                     w, h = (float(v) for v in mobj.groups())
3838                     if w > 0 and h > 0:
3839                         ratio = w / h
3840                         for f in formats:
3841                             if f.get('vcodec') != 'none':
3842                                 f['stretched_ratio'] = ratio
3843                         break
3844         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3845         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3846         if thumbnail_url:
3847             thumbnails.append({
3848                 'url': thumbnail_url,
3849             })
3850         original_thumbnails = thumbnails.copy()
3851
3852         # The best resolution thumbnails sometimes does not appear in the webpage
3853         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3854         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3855         thumbnail_names = [
3856             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3857             # in resolution, these are not the custom thumbnail. So de-prioritize them
3858             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3859             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3860         ]
3861         n_thumbnail_names = len(thumbnail_names)
3862         thumbnails.extend({
3863             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3864                 video_id=video_id, name=name, ext=ext,
3865                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
3866         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3867         for thumb in thumbnails:
3868             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3869             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3870         self._remove_duplicate_formats(thumbnails)
3871         self._downloader._sort_thumbnails(original_thumbnails)
3872
3873         category = get_first(microformats, 'category') or search_meta('genre')
3874         channel_id = str_or_none(
3875             get_first(video_details, 'channelId')
3876             or get_first(microformats, 'externalChannelId')
3877             or search_meta('channelId'))
3878         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3879
3880         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3881         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3882         if not duration and live_end_time and live_start_time:
3883             duration = live_end_time - live_start_time
3884
3885         needs_live_processing = self._needs_live_processing(live_status, duration)
3886
3887         def is_bad_format(fmt):
3888             if needs_live_processing and not fmt.get('is_from_start'):
3889                 return True
3890             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
3891                     and fmt.get('protocol') == 'http_dash_segments'):
3892                 return True
3893
3894         for fmt in filter(is_bad_format, formats):
3895             fmt['preference'] = (fmt.get('preference') or -1) - 10
3896             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
3897
3898         if needs_live_processing:
3899             self._prepare_live_from_start_formats(
3900                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
3901
3902         formats.extend(self._extract_storyboard(player_responses, duration))
3903
3904         # source_preference is lower for throttled/potentially damaged formats
3905         self._sort_formats(formats, (
3906             'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto'))
3907
3908         info = {
3909             'id': video_id,
3910             'title': video_title,
3911             'formats': formats,
3912             'thumbnails': thumbnails,
3913             # The best thumbnail that we are sure exists. Prevents unnecessary
3914             # URL checking if user don't care about getting the best possible thumbnail
3915             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3916             'description': video_description,
3917             'uploader': get_first(video_details, 'author'),
3918             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3919             'uploader_url': owner_profile_url,
3920             'channel_id': channel_id,
3921             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3922             'duration': duration,
3923             'view_count': int_or_none(
3924                 get_first((video_details, microformats), (..., 'viewCount'))
3925                 or search_meta('interactionCount')),
3926             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3927             'age_limit': 18 if (
3928                 get_first(microformats, 'isFamilySafe') is False
3929                 or search_meta('isFamilyFriendly') == 'false'
3930                 or search_meta('og:restrictions:age') == '18+') else 0,
3931             'webpage_url': webpage_url,
3932             'categories': [category] if category else None,
3933             'tags': keywords,
3934             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3935             'live_status': live_status,
3936             'release_timestamp': live_start_time,
3937         }
3938
3939         subtitles = {}
3940         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3941         if pctr:
3942             def get_lang_code(track):
3943                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3944                         or track.get('languageCode'))
3945
3946             # Converted into dicts to remove duplicates
3947             captions = {
3948                 get_lang_code(sub): sub
3949                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3950             translation_languages = {
3951                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3952                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3953
3954             def process_language(container, base_url, lang_code, sub_name, query):
3955                 lang_subs = container.setdefault(lang_code, [])
3956                 for fmt in self._SUBTITLE_FORMATS:
3957                     query.update({
3958                         'fmt': fmt,
3959                     })
3960                     lang_subs.append({
3961                         'ext': fmt,
3962                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3963                         'name': sub_name,
3964                     })
3965
3966             # NB: Constructing the full subtitle dictionary is slow
3967             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
3968                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
3969             for lang_code, caption_track in captions.items():
3970                 base_url = caption_track.get('baseUrl')
3971                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3972                 if not base_url:
3973                     continue
3974                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3975                 if caption_track.get('kind') != 'asr':
3976                     if not lang_code:
3977                         continue
3978                     process_language(
3979                         subtitles, base_url, lang_code, lang_name, {})
3980                     if not caption_track.get('isTranslatable'):
3981                         continue
3982                 for trans_code, trans_name in translation_languages.items():
3983                     if not trans_code:
3984                         continue
3985                     orig_trans_code = trans_code
3986                     if caption_track.get('kind') != 'asr':
3987                         if not get_translated_subs:
3988                             continue
3989                         trans_code += f'-{lang_code}'
3990                         trans_name += format_field(lang_name, None, ' from %s')
3991                     # Add an "-orig" label to the original language so that it can be distinguished.
3992                     # The subs are returned without "-orig" as well for compatibility
3993                     if lang_code == f'a-{orig_trans_code}':
3994                         process_language(
3995                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3996                     # Setting tlang=lang returns damaged subtitles.
3997                     process_language(automatic_captions, base_url, trans_code, trans_name,
3998                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3999
4000         info['automatic_captions'] = automatic_captions
4001         info['subtitles'] = subtitles
4002
4003         parsed_url = urllib.parse.urlparse(url)
4004         for component in [parsed_url.fragment, parsed_url.query]:
4005             query = urllib.parse.parse_qs(component)
4006             for k, v in query.items():
4007                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4008                     d_k += '_time'
4009                     if d_k not in info and k in s_ks:
4010                         info[d_k] = parse_duration(query[k][0])
4011
4012         # Youtube Music Auto-generated description
4013         if video_description:
4014             mobj = re.search(
4015                 r'''(?xs)
4016                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4017                     (?P<album>[^\n]+)
4018                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4019                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4020                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4021                     .+\nAuto-generated\ by\ YouTube\.\s*$
4022                 ''', video_description)
4023             if mobj:
4024                 release_year = mobj.group('release_year')
4025                 release_date = mobj.group('release_date')
4026                 if release_date:
4027                     release_date = release_date.replace('-', '')
4028                     if not release_year:
4029                         release_year = release_date[:4]
4030                 info.update({
4031                     'album': mobj.group('album'.strip()),
4032                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4033                     'track': mobj.group('track').strip(),
4034                     'release_date': release_date,
4035                     'release_year': int_or_none(release_year),
4036                 })
4037
4038         initial_data = None
4039         if webpage:
4040             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4041         if not initial_data:
4042             query = {'videoId': video_id}
4043             query.update(self._get_checkok_params())
4044             initial_data = self._extract_response(
4045                 item_id=video_id, ep='next', fatal=False,
4046                 ytcfg=master_ytcfg, query=query,
4047                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4048                 note='Downloading initial data API JSON')
4049
4050         info['comment_count'] = traverse_obj(initial_data, (
4051             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4052             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
4053         ), (
4054             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4055             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
4056         ), expected_type=int_or_none, get_all=False)
4057
4058         try:  # This will error if there is no livechat
4059             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4060         except (KeyError, IndexError, TypeError):
4061             pass
4062         else:
4063             info.setdefault('subtitles', {})['live_chat'] = [{
4064                 # url is needed to set cookies
4065                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4066                 'video_id': video_id,
4067                 'ext': 'json',
4068                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4069                              else 'youtube_live_chat_replay'),
4070             }]
4071
4072         if initial_data:
4073             info['chapters'] = (
4074                 self._extract_chapters_from_json(initial_data, duration)
4075                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4076                 or self._extract_chapters_from_description(video_description, duration)
4077                 or None)
4078
4079         contents = traverse_obj(
4080             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4081             expected_type=list, default=[])
4082
4083         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4084         if vpir:
4085             stl = vpir.get('superTitleLink')
4086             if stl:
4087                 stl = self._get_text(stl)
4088                 if try_get(
4089                         vpir,
4090                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4091                     info['location'] = stl
4092                 else:
4093                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4094                     if mobj:
4095                         info.update({
4096                             'series': mobj.group(1),
4097                             'season_number': int(mobj.group(2)),
4098                             'episode_number': int(mobj.group(3)),
4099                         })
4100             for tlb in (try_get(
4101                     vpir,
4102                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4103                     list) or []):
4104                 tbrs = variadic(
4105                     traverse_obj(
4106                         tlb, 'toggleButtonRenderer',
4107                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer'),
4108                         default=[]))
4109                 for tbr in tbrs:
4110                     for getter, regex in [(
4111                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4112                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4113                                 lambda x: x['accessibility'],
4114                                 lambda x: x['accessibilityData']['accessibilityData'],
4115                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4116                         label = (try_get(tbr, getter, dict) or {}).get('label')
4117                         if label:
4118                             mobj = re.match(regex, label)
4119                             if mobj:
4120                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4121                                 break
4122             sbr_tooltip = try_get(
4123                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4124             if sbr_tooltip:
4125                 like_count, dislike_count = sbr_tooltip.split(' / ')
4126                 info.update({
4127                     'like_count': str_to_int(like_count),
4128                     'dislike_count': str_to_int(dislike_count),
4129                 })
4130             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4131             if vcr:
4132                 vc = self._get_count(vcr, 'viewCount')
4133                 # Upcoming premieres with waiting count are treated as live here
4134                 if vcr.get('isLive'):
4135                     info['concurrent_view_count'] = vc
4136                 elif info.get('view_count') is None:
4137                     info['view_count'] = vc
4138
4139         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4140         if vsir:
4141             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4142             info.update({
4143                 'channel': self._get_text(vor, 'title'),
4144                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4145
4146             rows = try_get(
4147                 vsir,
4148                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4149                 list) or []
4150             multiple_songs = False
4151             for row in rows:
4152                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4153                     multiple_songs = True
4154                     break
4155             for row in rows:
4156                 mrr = row.get('metadataRowRenderer') or {}
4157                 mrr_title = mrr.get('title')
4158                 if not mrr_title:
4159                     continue
4160                 mrr_title = self._get_text(mrr, 'title')
4161                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4162                 if mrr_title == 'License':
4163                     info['license'] = mrr_contents_text
4164                 elif not multiple_songs:
4165                     if mrr_title == 'Album':
4166                         info['album'] = mrr_contents_text
4167                     elif mrr_title == 'Artist':
4168                         info['artist'] = mrr_contents_text
4169                     elif mrr_title == 'Song':
4170                         info['track'] = mrr_contents_text
4171
4172         fallbacks = {
4173             'channel': 'uploader',
4174             'channel_id': 'uploader_id',
4175             'channel_url': 'uploader_url',
4176         }
4177
4178         # The upload date for scheduled, live and past live streams / premieres in microformats
4179         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4180         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4181         upload_date = (
4182             unified_strdate(get_first(microformats, 'uploadDate'))
4183             or unified_strdate(search_meta('uploadDate')))
4184         if not upload_date or (
4185             live_status in ('not_live', None)
4186             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4187         ):
4188             upload_date = strftime_or_none(
4189                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4190         info['upload_date'] = upload_date
4191
4192         for to, frm in fallbacks.items():
4193             if not info.get(to):
4194                 info[to] = info.get(frm)
4195
4196         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4197             v = info.get(s_k)
4198             if v:
4199                 info[d_k] = v
4200
4201         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4202
4203         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4204                       or get_first(video_details, 'isPrivate', expected_type=bool))
4205
4206         info['availability'] = (
4207             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4208             else self._availability(
4209                 is_private=is_private,
4210                 needs_premium=(
4211                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4212                     or False if initial_data and is_private is not None else None),
4213                 needs_subscription=(
4214                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4215                     or False if initial_data and is_private is not None else None),
4216                 needs_auth=info['age_limit'] >= 18,
4217                 is_unlisted=None if is_private is None else (
4218                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4219                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4220
4221         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4222
4223         self.mark_watched(video_id, player_responses)
4224
4225         return info
4226
4227
4228 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4229
4230     @staticmethod
4231     def passthrough_smuggled_data(func):
4232         def _smuggle(entries, smuggled_data):
4233             for entry in entries:
4234                 # TODO: Convert URL to music.youtube instead.
4235                 # Do we need to passthrough any other smuggled_data?
4236                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
4237                 yield entry
4238
4239         @functools.wraps(func)
4240         def wrapper(self, url):
4241             url, smuggled_data = unsmuggle_url(url, {})
4242             if self.is_music_url(url):
4243                 smuggled_data['is_music_url'] = True
4244             info_dict = func(self, url, smuggled_data)
4245             if smuggled_data and info_dict.get('entries'):
4246                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
4247             return info_dict
4248         return wrapper
4249
4250     def _extract_channel_id(self, webpage):
4251         channel_id = self._html_search_meta(
4252             'channelId', webpage, 'channel id', default=None)
4253         if channel_id:
4254             return channel_id
4255         channel_url = self._html_search_meta(
4256             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
4257              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
4258              'twitter:app:url:googleplay'), webpage, 'channel url')
4259         return self._search_regex(
4260             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
4261             channel_url, 'channel id')
4262
4263     @staticmethod
4264     def _extract_basic_item_renderer(item):
4265         # Modified from _extract_grid_item_renderer
4266         known_basic_renderers = (
4267             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4268         )
4269         for key, renderer in item.items():
4270             if not isinstance(renderer, dict):
4271                 continue
4272             elif key in known_basic_renderers:
4273                 return renderer
4274             elif key.startswith('grid') and key.endswith('Renderer'):
4275                 return renderer
4276
4277     def _grid_entries(self, grid_renderer):
4278         for item in grid_renderer['items']:
4279             if not isinstance(item, dict):
4280                 continue
4281             renderer = self._extract_basic_item_renderer(item)
4282             if not isinstance(renderer, dict):
4283                 continue
4284             title = self._get_text(renderer, 'title')
4285
4286             # playlist
4287             playlist_id = renderer.get('playlistId')
4288             if playlist_id:
4289                 yield self.url_result(
4290                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4291                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4292                     video_title=title)
4293                 continue
4294             # video
4295             video_id = renderer.get('videoId')
4296             if video_id:
4297                 yield self._extract_video(renderer)
4298                 continue
4299             # channel
4300             channel_id = renderer.get('channelId')
4301             if channel_id:
4302                 yield self.url_result(
4303                     'https://www.youtube.com/channel/%s' % channel_id,
4304                     ie=YoutubeTabIE.ie_key(), video_title=title)
4305                 continue
4306             # generic endpoint URL support
4307             ep_url = urljoin('https://www.youtube.com/', try_get(
4308                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4309                 str))
4310             if ep_url:
4311                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4312                     if ie.suitable(ep_url):
4313                         yield self.url_result(
4314                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4315                         break
4316
4317     def _music_reponsive_list_entry(self, renderer):
4318         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4319         if video_id:
4320             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4321                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4322         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4323         if playlist_id:
4324             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4325             if video_id:
4326                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4327                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4328             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4329                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4330         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4331         if browse_id:
4332             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4333                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4334
4335     def _shelf_entries_from_content(self, shelf_renderer):
4336         content = shelf_renderer.get('content')
4337         if not isinstance(content, dict):
4338             return
4339         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4340         if renderer:
4341             # TODO: add support for nested playlists so each shelf is processed
4342             # as separate playlist
4343             # TODO: this includes only first N items
4344             yield from self._grid_entries(renderer)
4345         renderer = content.get('horizontalListRenderer')
4346         if renderer:
4347             # TODO
4348             pass
4349
4350     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4351         ep = try_get(
4352             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4353             str)
4354         shelf_url = urljoin('https://www.youtube.com', ep)
4355         if shelf_url:
4356             # Skipping links to another channels, note that checking for
4357             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4358             # will not work
4359             if skip_channels and '/channels?' in shelf_url:
4360                 return
4361             title = self._get_text(shelf_renderer, 'title')
4362             yield self.url_result(shelf_url, video_title=title)
4363         # Shelf may not contain shelf URL, fallback to extraction from content
4364         yield from self._shelf_entries_from_content(shelf_renderer)
4365
4366     def _playlist_entries(self, video_list_renderer):
4367         for content in video_list_renderer['contents']:
4368             if not isinstance(content, dict):
4369                 continue
4370             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4371             if not isinstance(renderer, dict):
4372                 continue
4373             video_id = renderer.get('videoId')
4374             if not video_id:
4375                 continue
4376             yield self._extract_video(renderer)
4377
4378     def _rich_entries(self, rich_grid_renderer):
4379         renderer = traverse_obj(
4380             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4381         video_id = renderer.get('videoId')
4382         if not video_id:
4383             return
4384         yield self._extract_video(renderer)
4385
4386     def _video_entry(self, video_renderer):
4387         video_id = video_renderer.get('videoId')
4388         if video_id:
4389             return self._extract_video(video_renderer)
4390
4391     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4392         url = urljoin('https://youtube.com', traverse_obj(
4393             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4394         if url:
4395             return self.url_result(
4396                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4397
4398     def _post_thread_entries(self, post_thread_renderer):
4399         post_renderer = try_get(
4400             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4401         if not post_renderer:
4402             return
4403         # video attachment
4404         video_renderer = try_get(
4405             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4406         video_id = video_renderer.get('videoId')
4407         if video_id:
4408             entry = self._extract_video(video_renderer)
4409             if entry:
4410                 yield entry
4411         # playlist attachment
4412         playlist_id = try_get(
4413             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4414         if playlist_id:
4415             yield self.url_result(
4416                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4417                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4418         # inline video links
4419         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4420         for run in runs:
4421             if not isinstance(run, dict):
4422                 continue
4423             ep_url = try_get(
4424                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4425             if not ep_url:
4426                 continue
4427             if not YoutubeIE.suitable(ep_url):
4428                 continue
4429             ep_video_id = YoutubeIE._match_id(ep_url)
4430             if video_id == ep_video_id:
4431                 continue
4432             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4433
4434     def _post_thread_continuation_entries(self, post_thread_continuation):
4435         contents = post_thread_continuation.get('contents')
4436         if not isinstance(contents, list):
4437             return
4438         for content in contents:
4439             renderer = content.get('backstagePostThreadRenderer')
4440             if isinstance(renderer, dict):
4441                 yield from self._post_thread_entries(renderer)
4442                 continue
4443             renderer = content.get('videoRenderer')
4444             if isinstance(renderer, dict):
4445                 yield self._video_entry(renderer)
4446
4447     r''' # unused
4448     def _rich_grid_entries(self, contents):
4449         for content in contents:
4450             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4451             if video_renderer:
4452                 entry = self._video_entry(video_renderer)
4453                 if entry:
4454                     yield entry
4455     '''
4456
4457     def _report_history_entries(self, renderer):
4458         for url in traverse_obj(renderer, (
4459                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4460                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4461                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4462             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4463
4464     def _extract_entries(self, parent_renderer, continuation_list):
4465         # continuation_list is modified in-place with continuation_list = [continuation_token]
4466         continuation_list[:] = [None]
4467         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4468         for content in contents:
4469             if not isinstance(content, dict):
4470                 continue
4471             is_renderer = traverse_obj(
4472                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4473                 expected_type=dict)
4474             if not is_renderer:
4475                 if content.get('richItemRenderer'):
4476                     for entry in self._rich_entries(content['richItemRenderer']):
4477                         yield entry
4478                     continuation_list[0] = self._extract_continuation(parent_renderer)
4479                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4480                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4481                     yield from self._report_history_entries(table)
4482                     continuation_list[0] = self._extract_continuation(table)
4483                 continue
4484
4485             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4486             for isr_content in isr_contents:
4487                 if not isinstance(isr_content, dict):
4488                     continue
4489
4490                 known_renderers = {
4491                     'playlistVideoListRenderer': self._playlist_entries,
4492                     'gridRenderer': self._grid_entries,
4493                     'reelShelfRenderer': self._grid_entries,
4494                     'shelfRenderer': self._shelf_entries,
4495                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4496                     'backstagePostThreadRenderer': self._post_thread_entries,
4497                     'videoRenderer': lambda x: [self._video_entry(x)],
4498                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4499                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4500                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4501                 }
4502                 for key, renderer in isr_content.items():
4503                     if key not in known_renderers:
4504                         continue
4505                     for entry in known_renderers[key](renderer):
4506                         if entry:
4507                             yield entry
4508                     continuation_list[0] = self._extract_continuation(renderer)
4509                     break
4510
4511             if not continuation_list[0]:
4512                 continuation_list[0] = self._extract_continuation(is_renderer)
4513
4514         if not continuation_list[0]:
4515             continuation_list[0] = self._extract_continuation(parent_renderer)
4516
4517     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4518         continuation_list = [None]
4519         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4520         tab_content = try_get(tab, lambda x: x['content'], dict)
4521         if not tab_content:
4522             return
4523         parent_renderer = (
4524             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4525             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4526         yield from extract_entries(parent_renderer)
4527         continuation = continuation_list[0]
4528
4529         for page_num in itertools.count(1):
4530             if not continuation:
4531                 break
4532             headers = self.generate_api_headers(
4533                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4534             response = self._extract_response(
4535                 item_id=f'{item_id} page {page_num}',
4536                 query=continuation, headers=headers, ytcfg=ytcfg,
4537                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4538
4539             if not response:
4540                 break
4541             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4542             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4543             visitor_data = self._extract_visitor_data(response) or visitor_data
4544
4545             known_renderers = {
4546                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4547                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4548                 'gridVideoRenderer': (self._grid_entries, 'items'),
4549                 'gridChannelRenderer': (self._grid_entries, 'items'),
4550                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4551                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4552                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4553                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4554                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4555                 'playlistVideoListContinuation': (self._playlist_entries, None),
4556                 'gridContinuation': (self._grid_entries, None),
4557                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4558                 'sectionListContinuation': (extract_entries, None),  # for feeds
4559             }
4560
4561             continuation_items = traverse_obj(response, (
4562                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4563                 'appendContinuationItemsAction', 'continuationItems'
4564             ), 'continuationContents', get_all=False)
4565             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4566
4567             video_items_renderer = None
4568             for key in continuation_item.keys():
4569                 if key not in known_renderers:
4570                     continue
4571                 func, parent_key = known_renderers[key]
4572                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4573                 continuation_list = [None]
4574                 yield from func(video_items_renderer)
4575                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4576
4577             if not video_items_renderer:
4578                 break
4579
4580     @staticmethod
4581     def _extract_selected_tab(tabs, fatal=True):
4582         for tab in tabs:
4583             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4584             if renderer.get('selected') is True:
4585                 return renderer
4586         else:
4587             if fatal:
4588                 raise ExtractorError('Unable to find selected tab')
4589
4590     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4591         playlist_id = title = description = channel_url = channel_name = channel_id = None
4592         tags = []
4593
4594         selected_tab = self._extract_selected_tab(tabs)
4595         # Deprecated - remove when layout discontinued
4596         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4597         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
4598         metadata_renderer = try_get(
4599             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4600         if metadata_renderer:
4601             channel_name = metadata_renderer.get('title')
4602             channel_url = metadata_renderer.get('channelUrl')
4603             channel_id = metadata_renderer.get('externalId')
4604         else:
4605             metadata_renderer = try_get(
4606                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4607
4608         if metadata_renderer:
4609             title = metadata_renderer.get('title')
4610             description = metadata_renderer.get('description', '')
4611             playlist_id = channel_id
4612             tags = metadata_renderer.get('keywords', '').split()
4613
4614         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4615         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4616         def _get_uncropped(url):
4617             return url_or_none((url or '').split('=')[0] + '=s0')
4618
4619         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
4620         if avatar_thumbnails:
4621             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4622             if uncropped_avatar:
4623                 avatar_thumbnails.append({
4624                     'url': uncropped_avatar,
4625                     'id': 'avatar_uncropped',
4626                     'preference': 1
4627                 })
4628
4629         channel_banners = self._extract_thumbnails(
4630             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4631         for banner in channel_banners:
4632             banner['preference'] = -10
4633
4634         if channel_banners:
4635             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4636             if uncropped_banner:
4637                 channel_banners.append({
4638                     'url': uncropped_banner,
4639                     'id': 'banner_uncropped',
4640                     'preference': -5
4641                 })
4642
4643         # Deprecated - remove when old layout is discontinued
4644         primary_thumbnails = self._extract_thumbnails(
4645             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4646
4647         playlist_thumbnails = self._extract_thumbnails(
4648             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
4649
4650         if playlist_id is None:
4651             playlist_id = item_id
4652
4653         # Deprecated - remove primary_sidebar_renderer when old layout discontinued
4654         # Playlist stats is a text runs array containing [video count, view count, last updated].
4655         # last updated or (view count and last updated) may be missing.
4656         playlist_stats = get_first(
4657             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'),))
4658         last_updated_unix = self._parse_time_text(
4659             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
4660             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
4661
4662         view_count = self._get_count(playlist_stats, 1)
4663         if view_count is None:
4664             view_count = self._get_count(playlist_header_renderer, 'viewCountText')
4665
4666         playlist_count = self._get_count(playlist_stats, 0)
4667         if playlist_count is None:
4668             playlist_count = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
4669
4670         if title is None:
4671             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4672         title += format_field(selected_tab, 'title', ' - %s')
4673         title += format_field(selected_tab, 'expandedText', ' - %s')
4674
4675         metadata = {
4676             'playlist_id': playlist_id,
4677             'playlist_title': title,
4678             'playlist_description': description,
4679             'uploader': channel_name,
4680             'uploader_id': channel_id,
4681             'uploader_url': channel_url,
4682             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
4683             'tags': tags,
4684             'view_count': view_count,
4685             'availability': self._extract_availability(data),
4686             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4687             'playlist_count': playlist_count,
4688             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4689         }
4690         if not channel_id:
4691             owner = traverse_obj(playlist_header_renderer, 'ownerText')
4692             if not owner:
4693                 # Deprecated
4694                 owner = traverse_obj(
4695                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
4696                     ('videoOwner', 'videoOwnerRenderer', 'title'))
4697             owner_text = self._get_text(owner)
4698             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
4699             metadata.update(filter_dict({
4700                 'uploader': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
4701                 'uploader_id': browse_ep.get('browseId'),
4702                 'uploader_url': urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))
4703             }))
4704
4705         metadata.update({
4706             'channel': metadata['uploader'],
4707             'channel_id': metadata['uploader_id'],
4708             'channel_url': metadata['uploader_url']})
4709         return self.playlist_result(
4710             self._entries(
4711                 selected_tab, playlist_id, ytcfg,
4712                 self._extract_account_syncid(ytcfg, data),
4713                 self._extract_visitor_data(data, ytcfg)),
4714             **metadata)
4715
4716     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4717         first_id = last_id = response = None
4718         for page_num in itertools.count(1):
4719             videos = list(self._playlist_entries(playlist))
4720             if not videos:
4721                 return
4722             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4723             if start >= len(videos):
4724                 return
4725             yield from videos[start:]
4726             first_id = first_id or videos[0]['id']
4727             last_id = videos[-1]['id']
4728             watch_endpoint = try_get(
4729                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4730             headers = self.generate_api_headers(
4731                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4732                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4733             query = {
4734                 'playlistId': playlist_id,
4735                 'videoId': watch_endpoint.get('videoId') or last_id,
4736                 'index': watch_endpoint.get('index') or len(videos),
4737                 'params': watch_endpoint.get('params') or 'OAE%3D'
4738             }
4739             response = self._extract_response(
4740                 item_id='%s page %d' % (playlist_id, page_num),
4741                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4742                 check_get_keys='contents'
4743             )
4744             playlist = try_get(
4745                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4746
4747     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4748         title = playlist.get('title') or try_get(
4749             data, lambda x: x['titleText']['simpleText'], str)
4750         playlist_id = playlist.get('playlistId') or item_id
4751
4752         # Delegating everything except mix playlists to regular tab-based playlist URL
4753         playlist_url = urljoin(url, try_get(
4754             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4755             str))
4756
4757         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4758         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4759         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4760
4761         if playlist_url and playlist_url != url and not is_known_unviewable:
4762             return self.url_result(
4763                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4764                 video_title=title)
4765
4766         return self.playlist_result(
4767             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4768             playlist_id=playlist_id, playlist_title=title)
4769
4770     def _extract_availability(self, data):
4771         """
4772         Gets the availability of a given playlist/tab.
4773         Note: Unless YouTube tells us explicitly, we do not assume it is public
4774         @param data: response
4775         """
4776         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4777         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
4778         player_header_privacy = playlist_header_renderer.get('privacy')
4779
4780         badges = self._extract_badges(sidebar_renderer)
4781
4782         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4783         privacy_setting_icon = get_first(
4784             (playlist_header_renderer, sidebar_renderer),
4785             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
4786              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
4787             expected_type=str)
4788
4789         microformats_is_unlisted = traverse_obj(
4790             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
4791
4792         return (
4793             'public' if (
4794                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4795                 or player_header_privacy == 'PUBLIC'
4796                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
4797             else self._availability(
4798                 is_private=(
4799                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4800                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
4801                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
4802                 is_unlisted=(
4803                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4804                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
4805                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
4806                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
4807                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
4808                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
4809                 needs_auth=False))
4810
4811     @staticmethod
4812     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4813         sidebar_renderer = try_get(
4814             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4815         for item in sidebar_renderer:
4816             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4817             if renderer:
4818                 return renderer
4819
4820     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4821         """
4822         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
4823         """
4824         is_playlist = bool(traverse_obj(
4825             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
4826         if not is_playlist:
4827             return
4828         headers = self.generate_api_headers(
4829             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4830             visitor_data=self._extract_visitor_data(data, ytcfg))
4831         query = {
4832             'params': 'wgYCCAA=',
4833             'browseId': f'VL{item_id}'
4834         }
4835         return self._extract_response(
4836             item_id=item_id, headers=headers, query=query,
4837             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4838             note='Redownloading playlist API JSON with unavailable videos')
4839
4840     @functools.cached_property
4841     def skip_webpage(self):
4842         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4843
4844     def _extract_webpage(self, url, item_id, fatal=True):
4845         webpage, data = None, None
4846         for retry in self.RetryManager(fatal=fatal):
4847             try:
4848                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
4849                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4850             except ExtractorError as e:
4851                 if isinstance(e.cause, network_exceptions):
4852                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4853                         retry.error = e
4854                         continue
4855                 self._error_or_warning(e, fatal=fatal)
4856                 break
4857
4858             try:
4859                 self._extract_and_report_alerts(data)
4860             except ExtractorError as e:
4861                 self._error_or_warning(e, fatal=fatal)
4862                 break
4863
4864             # Sometimes youtube returns a webpage with incomplete ytInitialData
4865             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4866             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
4867                 retry.error = ExtractorError('Incomplete yt initial data received')
4868                 continue
4869
4870         return webpage, data
4871
4872     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4873         """Use if failed to extract ytcfg (and data) from initial webpage"""
4874         if not ytcfg and self.is_authenticated:
4875             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4876             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4877                 raise ExtractorError(
4878                     f'{msg}. If you are not downloading private content, or '
4879                     'your cookies are only for the first account and channel,'
4880                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4881                     expected=True)
4882             self.report_warning(msg, only_once=True)
4883
4884     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4885         data = None
4886         if not self.skip_webpage:
4887             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4888             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4889             # Reject webpage data if redirected to home page without explicitly requesting
4890             selected_tab = self._extract_selected_tab(traverse_obj(
4891                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4892             if (url != 'https://www.youtube.com/feed/recommended'
4893                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4894                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4895                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4896                 if fatal:
4897                     raise ExtractorError(msg, expected=True)
4898                 self.report_warning(msg, only_once=True)
4899         if not data:
4900             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4901             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4902         return data, ytcfg
4903
4904     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4905         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4906         resolve_response = self._extract_response(
4907             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4908             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4909         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4910         for ep_key, ep in endpoints.items():
4911             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4912             if params:
4913                 return self._extract_response(
4914                     item_id=item_id, query=params, ep=ep, headers=headers,
4915                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4916                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4917         err_note = 'Failed to resolve url (does the playlist exist?)'
4918         if fatal:
4919             raise ExtractorError(err_note, expected=True)
4920         self.report_warning(err_note, item_id)
4921
4922     _SEARCH_PARAMS = None
4923
4924     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4925         data = {'query': query}
4926         if params is NO_DEFAULT:
4927             params = self._SEARCH_PARAMS
4928         if params:
4929             data['params'] = params
4930
4931         content_keys = (
4932             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4933             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4934             # ytmusic search
4935             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4936             ('continuationContents', ),
4937         )
4938         display_id = f'query "{query}"'
4939         check_get_keys = tuple({keys[0] for keys in content_keys})
4940         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4941         self._report_playlist_authcheck(ytcfg, fatal=False)
4942
4943         continuation_list = [None]
4944         search = None
4945         for page_num in itertools.count(1):
4946             data.update(continuation_list[0] or {})
4947             headers = self.generate_api_headers(
4948                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4949             search = self._extract_response(
4950                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4951                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4952             slr_contents = traverse_obj(search, *content_keys)
4953             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4954             if not continuation_list[0]:
4955                 break
4956
4957
4958 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4959     IE_DESC = 'YouTube Tabs'
4960     _VALID_URL = r'''(?x:
4961         https?://
4962             (?:\w+\.)?
4963             (?:
4964                 youtube(?:kids)?\.com|
4965                 %(invidious)s
4966             )/
4967             (?:
4968                 (?P<channel_type>channel|c|user|browse)/|
4969                 (?P<not_channel>
4970                     feed/|hashtag/|
4971                     (?:playlist|watch)\?.*?\blist=
4972                 )|
4973                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4974             )
4975             (?P<id>[^/?\#&]+)
4976     )''' % {
4977         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4978         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4979     }
4980     IE_NAME = 'youtube:tab'
4981
4982     _TESTS = [{
4983         'note': 'playlists, multipage',
4984         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4985         'playlist_mincount': 94,
4986         'info_dict': {
4987             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4988             'title': 'Igor Kleiner - Playlists',
4989             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4990             'uploader': 'Igor Kleiner',
4991             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4992             'channel': 'Igor Kleiner',
4993             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4994             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4995             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4996             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4997             'channel_follower_count': int
4998         },
4999     }, {
5000         'note': 'playlists, multipage, different order',
5001         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5002         'playlist_mincount': 94,
5003         'info_dict': {
5004             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5005             'title': 'Igor Kleiner - Playlists',
5006             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5007             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5008             'uploader': 'Igor Kleiner',
5009             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5010             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5011             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5012             'channel': 'Igor Kleiner',
5013             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5014             'channel_follower_count': int
5015         },
5016     }, {
5017         'note': 'playlists, series',
5018         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5019         'playlist_mincount': 5,
5020         'info_dict': {
5021             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5022             'title': '3Blue1Brown - Playlists',
5023             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5024             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5025             'uploader': '3Blue1Brown',
5026             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5027             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5028             'channel': '3Blue1Brown',
5029             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5030             'tags': ['Mathematics'],
5031             'channel_follower_count': int
5032         },
5033     }, {
5034         'note': 'playlists, singlepage',
5035         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5036         'playlist_mincount': 4,
5037         'info_dict': {
5038             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5039             'title': 'ThirstForScience - Playlists',
5040             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5041             'uploader': 'ThirstForScience',
5042             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5043             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5044             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5045             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5046             'tags': 'count:13',
5047             'channel': 'ThirstForScience',
5048             'channel_follower_count': int
5049         }
5050     }, {
5051         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5052         'only_matching': True,
5053     }, {
5054         'note': 'basic, single video playlist',
5055         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5056         'info_dict': {
5057             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5058             'uploader': 'Sergey M.',
5059             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5060             'title': 'youtube-dl public playlist',
5061             'description': '',
5062             'tags': [],
5063             'view_count': int,
5064             'modified_date': '20201130',
5065             'channel': 'Sergey M.',
5066             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5067             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5068             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5069             'availability': 'public',
5070         },
5071         'playlist_count': 1,
5072     }, {
5073         'note': 'empty playlist',
5074         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5075         'info_dict': {
5076             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5077             'uploader': 'Sergey M.',
5078             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5079             'title': 'youtube-dl empty playlist',
5080             'tags': [],
5081             'channel': 'Sergey M.',
5082             'description': '',
5083             'modified_date': '20160902',
5084             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5085             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5086             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5087             'availability': 'public',
5088         },
5089         'playlist_count': 0,
5090     }, {
5091         'note': 'Home tab',
5092         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5093         'info_dict': {
5094             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5095             'title': 'lex will - Home',
5096             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5097             'uploader': 'lex will',
5098             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5099             'channel': 'lex will',
5100             'tags': ['bible', 'history', 'prophesy'],
5101             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5102             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5103             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5104             'channel_follower_count': int
5105         },
5106         'playlist_mincount': 2,
5107     }, {
5108         'note': 'Videos tab',
5109         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5110         'info_dict': {
5111             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5112             'title': 'lex will - Videos',
5113             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5114             'uploader': 'lex will',
5115             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5116             'tags': ['bible', 'history', 'prophesy'],
5117             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5118             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5119             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5120             'channel': 'lex will',
5121             'channel_follower_count': int
5122         },
5123         'playlist_mincount': 975,
5124     }, {
5125         'note': 'Videos tab, sorted by popular',
5126         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5127         'info_dict': {
5128             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5129             'title': 'lex will - Videos',
5130             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5131             'uploader': 'lex will',
5132             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5133             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5134             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5135             'channel': 'lex will',
5136             'tags': ['bible', 'history', 'prophesy'],
5137             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5138             'channel_follower_count': int
5139         },
5140         'playlist_mincount': 199,
5141     }, {
5142         'note': 'Playlists tab',
5143         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5144         'info_dict': {
5145             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5146             'title': 'lex will - Playlists',
5147             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5148             'uploader': 'lex will',
5149             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5150             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5151             'channel': 'lex will',
5152             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5153             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5154             'tags': ['bible', 'history', 'prophesy'],
5155             'channel_follower_count': int
5156         },
5157         'playlist_mincount': 17,
5158     }, {
5159         'note': 'Community tab',
5160         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5161         'info_dict': {
5162             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5163             'title': 'lex will - Community',
5164             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5165             'uploader': 'lex will',
5166             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5167             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5168             'channel': 'lex will',
5169             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5170             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5171             'tags': ['bible', 'history', 'prophesy'],
5172             'channel_follower_count': int
5173         },
5174         'playlist_mincount': 18,
5175     }, {
5176         'note': 'Channels tab',
5177         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5178         'info_dict': {
5179             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5180             'title': 'lex will - Channels',
5181             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5182             'uploader': 'lex will',
5183             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5184             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5185             'channel': 'lex will',
5186             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5187             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5188             'tags': ['bible', 'history', 'prophesy'],
5189             'channel_follower_count': int
5190         },
5191         'playlist_mincount': 12,
5192     }, {
5193         'note': 'Search tab',
5194         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5195         'playlist_mincount': 40,
5196         'info_dict': {
5197             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5198             'title': '3Blue1Brown - Search - linear algebra',
5199             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5200             'uploader': '3Blue1Brown',
5201             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5202             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5203             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5204             'tags': ['Mathematics'],
5205             'channel': '3Blue1Brown',
5206             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5207             'channel_follower_count': int
5208         },
5209     }, {
5210         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5211         'only_matching': True,
5212     }, {
5213         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5214         'only_matching': True,
5215     }, {
5216         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5217         'only_matching': True,
5218     }, {
5219         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5220         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5221         'info_dict': {
5222             'title': '29C3: Not my department',
5223             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5224             'uploader': 'Christiaan008',
5225             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5226             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5227             'tags': [],
5228             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5229             'view_count': int,
5230             'modified_date': '20150605',
5231             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5232             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5233             'channel': 'Christiaan008',
5234             'availability': 'public',
5235         },
5236         'playlist_count': 96,
5237     }, {
5238         'note': 'Large playlist',
5239         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5240         'info_dict': {
5241             'title': 'Uploads from Cauchemar',
5242             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5243             'uploader': 'Cauchemar',
5244             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5245             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5246             'tags': [],
5247             'modified_date': r're:\d{8}',
5248             'channel': 'Cauchemar',
5249             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5250             'view_count': int,
5251             'description': '',
5252             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5253             'availability': 'public',
5254         },
5255         'playlist_mincount': 1123,
5256         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5257     }, {
5258         'note': 'even larger playlist, 8832 videos',
5259         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5260         'only_matching': True,
5261     }, {
5262         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5263         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5264         'info_dict': {
5265             'title': 'Uploads from Interstellar Movie',
5266             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5267             'uploader': 'Interstellar Movie',
5268             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5269             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5270             'tags': [],
5271             'view_count': int,
5272             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5273             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5274             'channel': 'Interstellar Movie',
5275             'description': '',
5276             'modified_date': r're:\d{8}',
5277             'availability': 'public',
5278         },
5279         'playlist_mincount': 21,
5280     }, {
5281         'note': 'Playlist with "show unavailable videos" button',
5282         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5283         'info_dict': {
5284             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5285             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5286             'uploader': 'Phim Siêu Nhân Nhật Bản',
5287             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5288             'view_count': int,
5289             'channel': 'Phim Siêu Nhân Nhật Bản',
5290             'tags': [],
5291             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5292             'description': '',
5293             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5294             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5295             'modified_date': r're:\d{8}',
5296             'availability': 'public',
5297         },
5298         'playlist_mincount': 200,
5299         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5300     }, {
5301         'note': 'Playlist with unavailable videos in page 7',
5302         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5303         'info_dict': {
5304             'title': 'Uploads from BlankTV',
5305             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5306             'uploader': 'BlankTV',
5307             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5308             'channel': 'BlankTV',
5309             'channel_url': 'https://www.youtube.com/c/blanktv',
5310             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5311             'view_count': int,
5312             'tags': [],
5313             'uploader_url': 'https://www.youtube.com/c/blanktv',
5314             'modified_date': r're:\d{8}',
5315             'description': '',
5316             'availability': 'public',
5317         },
5318         'playlist_mincount': 1000,
5319         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5320     }, {
5321         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5322         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5323         'info_dict': {
5324             'title': 'Data Analysis with Dr Mike Pound',
5325             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5326             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5327             'uploader': 'Computerphile',
5328             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5329             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5330             'tags': [],
5331             'view_count': int,
5332             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5333             'channel_url': 'https://www.youtube.com/user/Computerphile',
5334             'channel': 'Computerphile',
5335             'availability': 'public',
5336             'modified_date': '20190712',
5337         },
5338         'playlist_mincount': 11,
5339     }, {
5340         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5341         'only_matching': True,
5342     }, {
5343         'note': 'Playlist URL that does not actually serve a playlist',
5344         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5345         'info_dict': {
5346             'id': 'FqZTN594JQw',
5347             'ext': 'webm',
5348             'title': "Smiley's People 01 detective, Adventure Series, Action",
5349             'uploader': 'STREEM',
5350             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5351             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5352             'upload_date': '20150526',
5353             'license': 'Standard YouTube License',
5354             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5355             'categories': ['People & Blogs'],
5356             'tags': list,
5357             'view_count': int,
5358             'like_count': int,
5359         },
5360         'params': {
5361             'skip_download': True,
5362         },
5363         'skip': 'This video is not available.',
5364         'add_ie': [YoutubeIE.ie_key()],
5365     }, {
5366         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5367         'only_matching': True,
5368     }, {
5369         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5370         'only_matching': True,
5371     }, {
5372         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5373         'info_dict': {
5374             'id': 'Wq15eF5vCbI',  # This will keep changing
5375             'ext': 'mp4',
5376             'title': str,
5377             'uploader': 'Sky News',
5378             'uploader_id': 'skynews',
5379             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5380             'upload_date': r're:\d{8}',
5381             'description': str,
5382             'categories': ['News & Politics'],
5383             'tags': list,
5384             'like_count': int,
5385             'release_timestamp': 1642502819,
5386             'channel': 'Sky News',
5387             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5388             'age_limit': 0,
5389             'view_count': int,
5390             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5391             'playable_in_embed': True,
5392             'release_date': '20220118',
5393             'availability': 'public',
5394             'live_status': 'is_live',
5395             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5396             'channel_follower_count': int
5397         },
5398         'params': {
5399             'skip_download': True,
5400         },
5401         'expected_warnings': ['Ignoring subtitle tracks found in '],
5402     }, {
5403         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5404         'info_dict': {
5405             'id': 'a48o2S1cPoo',
5406             'ext': 'mp4',
5407             'title': 'The Young Turks - Live Main Show',
5408             'uploader': 'The Young Turks',
5409             'uploader_id': 'TheYoungTurks',
5410             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5411             'upload_date': '20150715',
5412             'license': 'Standard YouTube License',
5413             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5414             'categories': ['News & Politics'],
5415             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5416             'like_count': int,
5417         },
5418         'params': {
5419             'skip_download': True,
5420         },
5421         'only_matching': True,
5422     }, {
5423         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5424         'only_matching': True,
5425     }, {
5426         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5427         'only_matching': True,
5428     }, {
5429         'note': 'A channel that is not live. Should raise error',
5430         'url': 'https://www.youtube.com/user/numberphile/live',
5431         'only_matching': True,
5432     }, {
5433         'url': 'https://www.youtube.com/feed/trending',
5434         'only_matching': True,
5435     }, {
5436         'url': 'https://www.youtube.com/feed/library',
5437         'only_matching': True,
5438     }, {
5439         'url': 'https://www.youtube.com/feed/history',
5440         'only_matching': True,
5441     }, {
5442         'url': 'https://www.youtube.com/feed/subscriptions',
5443         'only_matching': True,
5444     }, {
5445         'url': 'https://www.youtube.com/feed/watch_later',
5446         'only_matching': True,
5447     }, {
5448         'note': 'Recommended - redirects to home page.',
5449         'url': 'https://www.youtube.com/feed/recommended',
5450         'only_matching': True,
5451     }, {
5452         'note': 'inline playlist with not always working continuations',
5453         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5454         'only_matching': True,
5455     }, {
5456         'url': 'https://www.youtube.com/course',
5457         'only_matching': True,
5458     }, {
5459         'url': 'https://www.youtube.com/zsecurity',
5460         'only_matching': True,
5461     }, {
5462         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5463         'only_matching': True,
5464     }, {
5465         'url': 'https://www.youtube.com/TheYoungTurks/live',
5466         'only_matching': True,
5467     }, {
5468         'url': 'https://www.youtube.com/hashtag/cctv9',
5469         'info_dict': {
5470             'id': 'cctv9',
5471             'title': '#cctv9',
5472             'tags': [],
5473         },
5474         'playlist_mincount': 350,
5475     }, {
5476         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5477         'only_matching': True,
5478     }, {
5479         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5480         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5481         'only_matching': True
5482     }, {
5483         'note': '/browse/ should redirect to /channel/',
5484         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5485         'only_matching': True
5486     }, {
5487         'note': 'VLPL, should redirect to playlist?list=PL...',
5488         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5489         'info_dict': {
5490             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5491             'uploader': 'NoCopyrightSounds',
5492             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5493             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5494             'title': 'NCS : All Releases 💿',
5495             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5496             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5497             'modified_date': r're:\d{8}',
5498             'view_count': int,
5499             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5500             'tags': [],
5501             'channel': 'NoCopyrightSounds',
5502             'availability': 'public',
5503         },
5504         'playlist_mincount': 166,
5505         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5506     }, {
5507         'note': 'Topic, should redirect to playlist?list=UU...',
5508         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5509         'info_dict': {
5510             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5511             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5512             'title': 'Uploads from Royalty Free Music - Topic',
5513             'uploader': 'Royalty Free Music - Topic',
5514             'tags': [],
5515             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5516             'channel': 'Royalty Free Music - Topic',
5517             'view_count': int,
5518             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5519             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5520             'modified_date': r're:\d{8}',
5521             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5522             'description': '',
5523             'availability': 'public',
5524         },
5525         'expected_warnings': [
5526             'The URL does not have a videos tab',
5527             r'[Uu]navailable videos (are|will be) hidden',
5528         ],
5529         'playlist_mincount': 101,
5530     }, {
5531         'note': 'Topic without a UU playlist',
5532         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5533         'info_dict': {
5534             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5535             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5536             'tags': [],
5537         },
5538         'expected_warnings': [
5539             'the playlist redirect gave error',
5540         ],
5541         'playlist_mincount': 9,
5542     }, {
5543         'note': 'Youtube music Album',
5544         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5545         'info_dict': {
5546             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5547             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5548             'tags': [],
5549             'view_count': int,
5550             'description': '',
5551             'availability': 'unlisted',
5552             'modified_date': r're:\d{8}',
5553         },
5554         'playlist_count': 50,
5555     }, {
5556         'note': 'unlisted single video playlist',
5557         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5558         'info_dict': {
5559             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5560             'uploader': 'colethedj',
5561             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5562             'title': 'yt-dlp unlisted playlist test',
5563             'availability': 'unlisted',
5564             'tags': [],
5565             'modified_date': '20220418',
5566             'channel': 'colethedj',
5567             'view_count': int,
5568             'description': '',
5569             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5570             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5571             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5572         },
5573         'playlist_count': 1,
5574     }, {
5575         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5576         'url': 'https://www.youtube.com/feed/recommended',
5577         'info_dict': {
5578             'id': 'recommended',
5579             'title': 'recommended',
5580             'tags': [],
5581         },
5582         'playlist_mincount': 50,
5583         'params': {
5584             'skip_download': True,
5585             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5586         },
5587     }, {
5588         'note': 'API Fallback: /videos tab, sorted by oldest first',
5589         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5590         'info_dict': {
5591             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5592             'title': 'Cody\'sLab - Videos',
5593             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5594             'uploader': 'Cody\'sLab',
5595             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5596             'channel': 'Cody\'sLab',
5597             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5598             'tags': [],
5599             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5600             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5601             'channel_follower_count': int
5602         },
5603         'playlist_mincount': 650,
5604         'params': {
5605             'skip_download': True,
5606             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5607         },
5608     }, {
5609         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5610         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5611         'info_dict': {
5612             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5613             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5614             'title': 'Uploads from Royalty Free Music - Topic',
5615             'uploader': 'Royalty Free Music - Topic',
5616             'modified_date': r're:\d{8}',
5617             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5618             'description': '',
5619             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5620             'tags': [],
5621             'channel': 'Royalty Free Music - Topic',
5622             'view_count': int,
5623             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5624             'availability': 'public',
5625         },
5626         'expected_warnings': [
5627             'does not have a videos tab',
5628             r'[Uu]navailable videos (are|will be) hidden',
5629         ],
5630         'playlist_mincount': 101,
5631         'params': {
5632             'skip_download': True,
5633             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5634         },
5635     }, {
5636         'note': 'non-standard redirect to regional channel',
5637         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5638         'only_matching': True
5639     }, {
5640         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5641         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5642         'info_dict': {
5643             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5644             'modified_date': '20220407',
5645             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5646             'tags': [],
5647             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5648             'uploader': 'pukkandan',
5649             'availability': 'unlisted',
5650             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5651             'channel': 'pukkandan',
5652             'description': 'Test for collaborative playlist',
5653             'title': 'yt-dlp test - collaborative playlist',
5654             'view_count': int,
5655             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5656         },
5657         'playlist_mincount': 2
5658     }, {
5659         'note': 'translated tab name',
5660         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5661         'info_dict': {
5662             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5663             'tags': [],
5664             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5665             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5666             'description': '',
5667             'title': 'cole-dlp-test-acc - 再生リスト',
5668             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5669             'uploader': 'cole-dlp-test-acc',
5670             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5671             'channel': 'cole-dlp-test-acc',
5672             'channel_follower_count': int,
5673         },
5674         'playlist_mincount': 1,
5675         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5676         'expected_warnings': ['Preferring "ja"'],
5677     }, {
5678         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5679         'note': 'preferred lang set with playlist with translated video titles',
5680         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5681         'info_dict': {
5682             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5683             'tags': [],
5684             'view_count': int,
5685             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5686             'uploader': 'cole-dlp-test-acc',
5687             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5688             'channel': 'cole-dlp-test-acc',
5689             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5690             'description': 'test',
5691             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5692             'title': 'dlp test playlist',
5693             'availability': 'public',
5694         },
5695         'playlist_mincount': 1,
5696         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5697         'expected_warnings': ['Preferring "ja"'],
5698     }, {
5699         # shorts audio pivot for 2GtVksBMYFM.
5700         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5701         'info_dict': {
5702             'id': 'sfv_audio_pivot',
5703             'title': 'sfv_audio_pivot',
5704             'tags': [],
5705         },
5706         'playlist_mincount': 50,
5707
5708     }]
5709
5710     @classmethod
5711     def suitable(cls, url):
5712         return False if YoutubeIE.suitable(url) else super().suitable(url)
5713
5714     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5715
5716     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5717     def _real_extract(self, url, smuggled_data):
5718         item_id = self._match_id(url)
5719         url = urllib.parse.urlunparse(
5720             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5721         compat_opts = self.get_param('compat_opts', [])
5722
5723         def get_mobj(url):
5724             mobj = self._URL_RE.match(url).groupdict()
5725             mobj.update((k, '') for k, v in mobj.items() if v is None)
5726             return mobj
5727
5728         mobj, redirect_warning = get_mobj(url), None
5729         # Youtube returns incomplete data if tabname is not lower case
5730         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5731         if is_channel:
5732             if smuggled_data.get('is_music_url'):
5733                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5734                     item_id = item_id[2:]
5735                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5736                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5737                     mdata = self._extract_tab_endpoint(
5738                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5739                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5740                                         get_all=False, expected_type=str)
5741                     if not murl:
5742                         raise ExtractorError('Failed to resolve album to playlist')
5743                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5744                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5745                     pre = f'https://www.youtube.com/channel/{item_id}'
5746
5747         original_tab_name = tab
5748         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5749             # Home URLs should redirect to /videos/
5750             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5751                                 'To download only the videos in the home page, add a "/featured" to the URL')
5752             tab = '/videos'
5753
5754         url = ''.join((pre, tab, post))
5755         mobj = get_mobj(url)
5756
5757         # Handle both video/playlist URLs
5758         qs = parse_qs(url)
5759         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5760
5761         if not video_id and mobj['not_channel'].startswith('watch'):
5762             if not playlist_id:
5763                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5764                 raise ExtractorError('Unable to recognize tab page')
5765             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5766             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5767             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5768             mobj = get_mobj(url)
5769
5770         if video_id and playlist_id:
5771             if self.get_param('noplaylist'):
5772                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5773                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5774                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5775             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5776
5777         data, ytcfg = self._extract_data(url, item_id)
5778
5779         # YouTube may provide a non-standard redirect to the regional channel
5780         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5781         redirect_url = traverse_obj(
5782             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5783         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5784             redirect_url = ''.join((
5785                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5786             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5787             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5788
5789         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5790         if tabs:
5791             selected_tab = self._extract_selected_tab(tabs)
5792             selected_tab_url = urljoin(
5793                 url, traverse_obj(selected_tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
5794             translated_tab_name = selected_tab.get('title', '').lower()
5795
5796             # Prefer tab name from tab url as it is always in en,
5797             # but only when preferred lang is set as it may not extract reliably in all cases.
5798             selected_tab_name = (self._preferred_lang in (None, 'en') and translated_tab_name
5799                                  or selected_tab_url and get_mobj(selected_tab_url)['tab'][1:]  # primary
5800                                  or translated_tab_name)
5801
5802             if selected_tab_name == 'home':
5803                 selected_tab_name = 'featured'
5804             requested_tab_name = mobj['tab'][1:]
5805
5806             if 'no-youtube-channel-redirect' not in compat_opts:
5807                 if requested_tab_name == 'live':  # Live tab should have redirected to the video
5808                     raise UserNotLive(video_id=mobj['id'])
5809                 if requested_tab_name not in ('', selected_tab_name):
5810                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5811                     if not original_tab_name:
5812                         if item_id[:2] == 'UC':
5813                             # Topic channels don't have /videos. Use the equivalent playlist instead
5814                             pl_id = f'UU{item_id[2:]}'
5815                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5816                             try:
5817                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5818                             except ExtractorError:
5819                                 redirect_warning += ' and the playlist redirect gave error'
5820                             else:
5821                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5822                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5823                         if selected_tab_name and selected_tab_name != requested_tab_name:
5824                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5825                     else:
5826                         raise ExtractorError(redirect_warning, expected=True)
5827
5828         if redirect_warning:
5829             self.to_screen(redirect_warning)
5830         self.write_debug(f'Final URL: {url}')
5831
5832         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5833         if 'no-youtube-unavailable-videos' not in compat_opts:
5834             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5835         self._extract_and_report_alerts(data, only_once=True)
5836         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5837         if tabs:
5838             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5839
5840         playlist = traverse_obj(
5841             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5842         if playlist:
5843             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5844
5845         video_id = traverse_obj(
5846             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5847         if video_id:
5848             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5849                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5850             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5851                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5852
5853         raise ExtractorError('Unable to recognize tab page')
5854
5855
5856 class YoutubePlaylistIE(InfoExtractor):
5857     IE_DESC = 'YouTube playlists'
5858     _VALID_URL = r'''(?x)(?:
5859                         (?:https?://)?
5860                         (?:\w+\.)?
5861                         (?:
5862                             (?:
5863                                 youtube(?:kids)?\.com|
5864                                 %(invidious)s
5865                             )
5866                             /.*?\?.*?\blist=
5867                         )?
5868                         (?P<id>%(playlist_id)s)
5869                      )''' % {
5870         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5871         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5872     }
5873     IE_NAME = 'youtube:playlist'
5874     _TESTS = [{
5875         'note': 'issue #673',
5876         'url': 'PLBB231211A4F62143',
5877         'info_dict': {
5878             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5879             'id': 'PLBB231211A4F62143',
5880             'uploader': 'Wickman',
5881             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5882             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5883             'view_count': int,
5884             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5885             'modified_date': r're:\d{8}',
5886             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5887             'channel': 'Wickman',
5888             'tags': [],
5889             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5890         },
5891         'playlist_mincount': 29,
5892     }, {
5893         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5894         'info_dict': {
5895             'title': 'YDL_safe_search',
5896             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5897         },
5898         'playlist_count': 2,
5899         'skip': 'This playlist is private',
5900     }, {
5901         'note': 'embedded',
5902         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5903         'playlist_count': 4,
5904         'info_dict': {
5905             'title': 'JODA15',
5906             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5907             'uploader': 'milan',
5908             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5909             'description': '',
5910             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5911             'tags': [],
5912             'modified_date': '20140919',
5913             'view_count': int,
5914             'channel': 'milan',
5915             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5916             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5917             'availability': 'public',
5918         },
5919         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5920     }, {
5921         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5922         'playlist_mincount': 455,
5923         'info_dict': {
5924             'title': '2018 Chinese New Singles (11/6 updated)',
5925             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5926             'uploader': 'LBK',
5927             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5928             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5929             'channel': 'LBK',
5930             'view_count': int,
5931             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5932             'tags': [],
5933             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5934             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5935             'modified_date': r're:\d{8}',
5936             'availability': 'public',
5937         },
5938         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5939     }, {
5940         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5941         'only_matching': True,
5942     }, {
5943         # music album playlist
5944         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5945         'only_matching': True,
5946     }]
5947
5948     @classmethod
5949     def suitable(cls, url):
5950         if YoutubeTabIE.suitable(url):
5951             return False
5952         from ..utils import parse_qs
5953         qs = parse_qs(url)
5954         if qs.get('v', [None])[0]:
5955             return False
5956         return super().suitable(url)
5957
5958     def _real_extract(self, url):
5959         playlist_id = self._match_id(url)
5960         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5961         url = update_url_query(
5962             'https://www.youtube.com/playlist',
5963             parse_qs(url) or {'list': playlist_id})
5964         if is_music_url:
5965             url = smuggle_url(url, {'is_music_url': True})
5966         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5967
5968
5969 class YoutubeYtBeIE(InfoExtractor):
5970     IE_DESC = 'youtu.be'
5971     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5972     _TESTS = [{
5973         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5974         'info_dict': {
5975             'id': 'yeWKywCrFtk',
5976             'ext': 'mp4',
5977             'title': 'Small Scale Baler and Braiding Rugs',
5978             'uploader': 'Backus-Page House Museum',
5979             'uploader_id': 'backuspagemuseum',
5980             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5981             'upload_date': '20161008',
5982             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5983             'categories': ['Nonprofits & Activism'],
5984             'tags': list,
5985             'like_count': int,
5986             'age_limit': 0,
5987             'playable_in_embed': True,
5988             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5989             'channel': 'Backus-Page House Museum',
5990             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5991             'live_status': 'not_live',
5992             'view_count': int,
5993             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5994             'availability': 'public',
5995             'duration': 59,
5996             'comment_count': int,
5997             'channel_follower_count': int
5998         },
5999         'params': {
6000             'noplaylist': True,
6001             'skip_download': True,
6002         },
6003     }, {
6004         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6005         'only_matching': True,
6006     }]
6007
6008     def _real_extract(self, url):
6009         mobj = self._match_valid_url(url)
6010         video_id = mobj.group('id')
6011         playlist_id = mobj.group('playlist_id')
6012         return self.url_result(
6013             update_url_query('https://www.youtube.com/watch', {
6014                 'v': video_id,
6015                 'list': playlist_id,
6016                 'feature': 'youtu.be',
6017             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6018
6019
6020 class YoutubeLivestreamEmbedIE(InfoExtractor):
6021     IE_DESC = 'YouTube livestream embeds'
6022     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6023     _TESTS = [{
6024         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6025         'only_matching': True,
6026     }]
6027
6028     def _real_extract(self, url):
6029         channel_id = self._match_id(url)
6030         return self.url_result(
6031             f'https://www.youtube.com/channel/{channel_id}/live',
6032             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6033
6034
6035 class YoutubeYtUserIE(InfoExtractor):
6036     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6037     IE_NAME = 'youtube:user'
6038     _VALID_URL = r'ytuser:(?P<id>.+)'
6039     _TESTS = [{
6040         'url': 'ytuser:phihag',
6041         'only_matching': True,
6042     }]
6043
6044     def _real_extract(self, url):
6045         user_id = self._match_id(url)
6046         return self.url_result(
6047             'https://www.youtube.com/user/%s/videos' % user_id,
6048             ie=YoutubeTabIE.ie_key(), video_id=user_id)
6049
6050
6051 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6052     IE_NAME = 'youtube:favorites'
6053     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6054     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6055     _LOGIN_REQUIRED = True
6056     _TESTS = [{
6057         'url': ':ytfav',
6058         'only_matching': True,
6059     }, {
6060         'url': ':ytfavorites',
6061         'only_matching': True,
6062     }]
6063
6064     def _real_extract(self, url):
6065         return self.url_result(
6066             'https://www.youtube.com/playlist?list=LL',
6067             ie=YoutubeTabIE.ie_key())
6068
6069
6070 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6071     IE_NAME = 'youtube:notif'
6072     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6073     _VALID_URL = r':ytnotif(?:ication)?s?'
6074     _LOGIN_REQUIRED = True
6075     _TESTS = [{
6076         'url': ':ytnotif',
6077         'only_matching': True,
6078     }, {
6079         'url': ':ytnotifications',
6080         'only_matching': True,
6081     }]
6082
6083     def _extract_notification_menu(self, response, continuation_list):
6084         notification_list = traverse_obj(
6085             response,
6086             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6087             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6088             expected_type=list) or []
6089         continuation_list[0] = None
6090         for item in notification_list:
6091             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6092             if entry:
6093                 yield entry
6094             continuation = item.get('continuationItemRenderer')
6095             if continuation:
6096                 continuation_list[0] = continuation
6097
6098     def _extract_notification_renderer(self, notification):
6099         video_id = traverse_obj(
6100             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6101         url = f'https://www.youtube.com/watch?v={video_id}'
6102         channel_id = None
6103         if not video_id:
6104             browse_ep = traverse_obj(
6105                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6106             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6107             post_id = self._search_regex(
6108                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6109                 'post id', default=None)
6110             if not channel_id or not post_id:
6111                 return
6112             # The direct /post url redirects to this in the browser
6113             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6114
6115         channel = traverse_obj(
6116             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6117             expected_type=str)
6118         notification_title = self._get_text(notification, 'shortMessage')
6119         if notification_title:
6120             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6121         # TODO: handle recommended videos
6122         title = self._search_regex(
6123             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6124             'video title', default=None)
6125         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6126                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6127                      else None)
6128         return {
6129             '_type': 'url',
6130             'url': url,
6131             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6132             'video_id': video_id,
6133             'title': title,
6134             'channel_id': channel_id,
6135             'channel': channel,
6136             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6137             'timestamp': timestamp,
6138         }
6139
6140     def _notification_menu_entries(self, ytcfg):
6141         continuation_list = [None]
6142         response = None
6143         for page in itertools.count(1):
6144             ctoken = traverse_obj(
6145                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6146             response = self._extract_response(
6147                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6148                 ep='notification/get_notification_menu', check_get_keys='actions',
6149                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6150             yield from self._extract_notification_menu(response, continuation_list)
6151             if not continuation_list[0]:
6152                 break
6153
6154     def _real_extract(self, url):
6155         display_id = 'notifications'
6156         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6157         self._report_playlist_authcheck(ytcfg)
6158         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6159
6160
6161 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6162     IE_DESC = 'YouTube search'
6163     IE_NAME = 'youtube:search'
6164     _SEARCH_KEY = 'ytsearch'
6165     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6166     _TESTS = [{
6167         'url': 'ytsearch5:youtube-dl test video',
6168         'playlist_count': 5,
6169         'info_dict': {
6170             'id': 'youtube-dl test video',
6171             'title': 'youtube-dl test video',
6172         }
6173     }]
6174
6175
6176 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6177     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6178     _SEARCH_KEY = 'ytsearchdate'
6179     IE_DESC = 'YouTube search, newest videos first'
6180     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6181     _TESTS = [{
6182         'url': 'ytsearchdate5:youtube-dl test video',
6183         'playlist_count': 5,
6184         'info_dict': {
6185             'id': 'youtube-dl test video',
6186             'title': 'youtube-dl test video',
6187         }
6188     }]
6189
6190
6191 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6192     IE_DESC = 'YouTube search URLs with sorting and filter support'
6193     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6194     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6195     _TESTS = [{
6196         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6197         'playlist_mincount': 5,
6198         'info_dict': {
6199             'id': 'youtube-dl test video',
6200             'title': 'youtube-dl test video',
6201         }
6202     }, {
6203         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6204         'playlist_mincount': 5,
6205         'info_dict': {
6206             'id': 'python',
6207             'title': 'python',
6208         }
6209     }, {
6210         'url': 'https://www.youtube.com/results?search_query=%23cats',
6211         'playlist_mincount': 1,
6212         'info_dict': {
6213             'id': '#cats',
6214             'title': '#cats',
6215             # The test suite does not have support for nested playlists
6216             # 'entries': [{
6217             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6218             #     'title': '#cats',
6219             # }],
6220         },
6221     }, {
6222         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6223         'only_matching': True,
6224     }]
6225
6226     def _real_extract(self, url):
6227         qs = parse_qs(url)
6228         query = (qs.get('search_query') or qs.get('q'))[0]
6229         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6230
6231
6232 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6233     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6234     IE_NAME = 'youtube:music:search_url'
6235     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6236     _TESTS = [{
6237         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6238         'playlist_count': 16,
6239         'info_dict': {
6240             'id': 'royalty free music',
6241             'title': 'royalty free music',
6242         }
6243     }, {
6244         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6245         'playlist_mincount': 30,
6246         'info_dict': {
6247             'id': 'royalty free music - songs',
6248             'title': 'royalty free music - songs',
6249         },
6250         'params': {'extract_flat': 'in_playlist'}
6251     }, {
6252         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6253         'playlist_mincount': 30,
6254         'info_dict': {
6255             'id': 'royalty free music - community playlists',
6256             'title': 'royalty free music - community playlists',
6257         },
6258         'params': {'extract_flat': 'in_playlist'}
6259     }]
6260
6261     _SECTIONS = {
6262         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6263         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6264         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6265         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6266         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6267         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6268     }
6269
6270     def _real_extract(self, url):
6271         qs = parse_qs(url)
6272         query = (qs.get('search_query') or qs.get('q'))[0]
6273         params = qs.get('sp', (None,))[0]
6274         if params:
6275             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6276         else:
6277             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6278             params = self._SECTIONS.get(section)
6279             if not params:
6280                 section = None
6281         title = join_nonempty(query, section, delim=' - ')
6282         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6283
6284
6285 class YoutubeFeedsInfoExtractor(InfoExtractor):
6286     """
6287     Base class for feed extractors
6288     Subclasses must re-define the _FEED_NAME property.
6289     """
6290     _LOGIN_REQUIRED = True
6291     _FEED_NAME = 'feeds'
6292
6293     def _real_initialize(self):
6294         YoutubeBaseInfoExtractor._check_login_required(self)
6295
6296     @classproperty
6297     def IE_NAME(self):
6298         return f'youtube:{self._FEED_NAME}'
6299
6300     def _real_extract(self, url):
6301         return self.url_result(
6302             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6303
6304
6305 class YoutubeWatchLaterIE(InfoExtractor):
6306     IE_NAME = 'youtube:watchlater'
6307     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6308     _VALID_URL = r':ytwatchlater'
6309     _TESTS = [{
6310         'url': ':ytwatchlater',
6311         'only_matching': True,
6312     }]
6313
6314     def _real_extract(self, url):
6315         return self.url_result(
6316             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6317
6318
6319 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6320     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6321     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6322     _FEED_NAME = 'recommended'
6323     _LOGIN_REQUIRED = False
6324     _TESTS = [{
6325         'url': ':ytrec',
6326         'only_matching': True,
6327     }, {
6328         'url': ':ytrecommended',
6329         'only_matching': True,
6330     }, {
6331         'url': 'https://youtube.com',
6332         'only_matching': True,
6333     }]
6334
6335
6336 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6337     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6338     _VALID_URL = r':ytsub(?:scription)?s?'
6339     _FEED_NAME = 'subscriptions'
6340     _TESTS = [{
6341         'url': ':ytsubs',
6342         'only_matching': True,
6343     }, {
6344         'url': ':ytsubscriptions',
6345         'only_matching': True,
6346     }]
6347
6348
6349 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6350     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6351     _VALID_URL = r':ythis(?:tory)?'
6352     _FEED_NAME = 'history'
6353     _TESTS = [{
6354         'url': ':ythistory',
6355         'only_matching': True,
6356     }]
6357
6358
6359 class YoutubeStoriesIE(InfoExtractor):
6360     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6361     IE_NAME = 'youtube:stories'
6362     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6363     _TESTS = [{
6364         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6365         'only_matching': True,
6366     }]
6367
6368     def _real_extract(self, url):
6369         playlist_id = f'RLTD{self._match_id(url)}'
6370         return self.url_result(
6371             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6372             ie=YoutubeTabIE, video_id=playlist_id)
6373
6374
6375 class YoutubeShortsAudioPivotIE(InfoExtractor):
6376     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6377     IE_NAME = 'youtube:shorts:pivot:audio'
6378     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6379     _TESTS = [{
6380         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6381         'only_matching': True,
6382     }]
6383
6384     @staticmethod
6385     def _generate_audio_pivot_params(video_id):
6386         """
6387         Generates sfv_audio_pivot browse params for this video id
6388         """
6389         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6390         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6391
6392     def _real_extract(self, url):
6393         video_id = self._match_id(url)
6394         return self.url_result(
6395             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6396             ie=YoutubeTabIE)
6397
6398
6399 class YoutubeTruncatedURLIE(InfoExtractor):
6400     IE_NAME = 'youtube:truncated_url'
6401     IE_DESC = False  # Do not list
6402     _VALID_URL = r'''(?x)
6403         (?:https?://)?
6404         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6405         (?:watch\?(?:
6406             feature=[a-z_]+|
6407             annotation_id=annotation_[^&]+|
6408             x-yt-cl=[0-9]+|
6409             hl=[^&]*|
6410             t=[0-9]+
6411         )?
6412         |
6413             attribution_link\?a=[^&]+
6414         )
6415         $
6416     '''
6417
6418     _TESTS = [{
6419         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6420         'only_matching': True,
6421     }, {
6422         'url': 'https://www.youtube.com/watch?',
6423         'only_matching': True,
6424     }, {
6425         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6426         'only_matching': True,
6427     }, {
6428         'url': 'https://www.youtube.com/watch?feature=foo',
6429         'only_matching': True,
6430     }, {
6431         'url': 'https://www.youtube.com/watch?hl=en-GB',
6432         'only_matching': True,
6433     }, {
6434         'url': 'https://www.youtube.com/watch?t=2372',
6435         'only_matching': True,
6436     }]
6437
6438     def _real_extract(self, url):
6439         raise ExtractorError(
6440             'Did you forget to quote the URL? Remember that & is a meta '
6441             'character in most shells, so you want to put the URL in quotes, '
6442             'like  youtube-dl '
6443             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6444             ' or simply  youtube-dl BaW_jenozKc  .',
6445             expected=True)
6446
6447
6448 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6449     IE_NAME = 'youtube:clip'
6450     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6451     _TESTS = [{
6452         # FIXME: Other metadata should be extracted from the clip, not from the base video
6453         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6454         'info_dict': {
6455             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6456             'ext': 'mp4',
6457             'section_start': 29.0,
6458             'section_end': 39.7,
6459             'duration': 10.7,
6460             'age_limit': 0,
6461             'availability': 'public',
6462             'categories': ['Gaming'],
6463             'channel': 'Scott The Woz',
6464             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6465             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6466             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6467             'like_count': int,
6468             'playable_in_embed': True,
6469             'tags': 'count:17',
6470             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6471             'title': 'Mobile Games on Console - Scott The Woz',
6472             'upload_date': '20210920',
6473             'uploader': 'Scott The Woz',
6474             'uploader_id': 'scottthewoz',
6475             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6476             'view_count': int,
6477             'live_status': 'not_live',
6478             'channel_follower_count': int
6479         }
6480     }]
6481
6482     def _real_extract(self, url):
6483         clip_id = self._match_id(url)
6484         _, data = self._extract_webpage(url, clip_id)
6485
6486         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6487         if not video_id:
6488             raise ExtractorError('Unable to find video ID')
6489
6490         clip_data = traverse_obj(data, (
6491             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6492             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6493             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6494             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6495
6496         return {
6497             '_type': 'url_transparent',
6498             'url': f'https://www.youtube.com/watch?v={video_id}',
6499             'ie_key': YoutubeIE.ie_key(),
6500             'id': clip_id,
6501             'section_start': int(clip_data['startTimeMs']) / 1000,
6502             'section_end': int(clip_data['endTimeMs']) / 1000,
6503         }
6504
6505
6506 class YoutubeTruncatedIDIE(InfoExtractor):
6507     IE_NAME = 'youtube:truncated_id'
6508     IE_DESC = False  # Do not list
6509     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6510
6511     _TESTS = [{
6512         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6513         'only_matching': True,
6514     }]
6515
6516     def _real_extract(self, url):
6517         video_id = self._match_id(url)
6518         raise ExtractorError(
6519             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6520             expected=True)