yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16
  17 from .common import InfoExtractor, SearchInfoExtractor
  18 from ..compat import functools
  19 from ..compat import (
  20     compat_chr,
  21     compat_HTTPError,
  22     compat_parse_qs,
  23     compat_str,
  24     compat_urllib_parse_unquote_plus,
  25     compat_urllib_parse_urlencode,
  26     compat_urllib_parse_urlparse,
  27     compat_urlparse,
  28 )
  29 from ..jsinterp import JSInterpreter
  30 from ..utils import (
  31     NO_DEFAULT,
  32     ExtractorError,
  33     bug_reports_message,
  34     classproperty,
  35     clean_html,
  36     datetime_from_str,
  37     dict_get,
  38     error_to_compat_str,
  39     float_or_none,
  40     format_field,
  41     get_first,
  42     int_or_none,
  43     is_html,
  44     join_nonempty,
  45     js_to_json,
  46     mimetype2ext,
  47     network_exceptions,
  48     orderedSet,
  49     parse_codecs,
  50     parse_count,
  51     parse_duration,
  52     parse_iso8601,
  53     parse_qs,
  54     qualities,
  55     remove_end,
  56     remove_start,
  57     smuggle_url,
  58     str_or_none,
  59     str_to_int,
  60     strftime_or_none,
  61     traverse_obj,
  62     try_get,
  63     unescapeHTML,
  64     unified_strdate,
  65     unified_timestamp,
  66     unsmuggle_url,
  67     update_url_query,
  68     url_or_none,
  69     urljoin,
  70     variadic,
  71 )
  72
  73 # any clients starting with _ cannot be explicity requested by the user
  74 INNERTUBE_CLIENTS = {
  75     'web': {
  76         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  77         'INNERTUBE_CONTEXT': {
  78             'client': {
  79                 'clientName': 'WEB',
  80                 'clientVersion': '2.20211221.00.00',
  81             }
  82         },
  83         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  84     },
  85     'web_embedded': {
  86         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  87         'INNERTUBE_CONTEXT': {
  88             'client': {
  89                 'clientName': 'WEB_EMBEDDED_PLAYER',
  90                 'clientVersion': '1.20211215.00.01',
  91             },
  92         },
  93         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  94     },
  95     'web_music': {
  96         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  97         'INNERTUBE_HOST': 'music.youtube.com',
  98         'INNERTUBE_CONTEXT': {
  99             'client': {
 100                 'clientName': 'WEB_REMIX',
 101                 'clientVersion': '1.20211213.00.00',
 102             }
 103         },
 104         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 105     },
 106     'web_creator': {
 107         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 108         'INNERTUBE_CONTEXT': {
 109             'client': {
 110                 'clientName': 'WEB_CREATOR',
 111                 'clientVersion': '1.20211220.02.00',
 112             }
 113         },
 114         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 115     },
 116     'android': {
 117         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 118         'INNERTUBE_CONTEXT': {
 119             'client': {
 120                 'clientName': 'ANDROID',
 121                 'clientVersion': '16.49',
 122             }
 123         },
 124         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 125         'REQUIRE_JS_PLAYER': False
 126     },
 127     'android_embedded': {
 128         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 129         'INNERTUBE_CONTEXT': {
 130             'client': {
 131                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 132                 'clientVersion': '16.49',
 133             },
 134         },
 135         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 136         'REQUIRE_JS_PLAYER': False
 137     },
 138     'android_music': {
 139         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 140         'INNERTUBE_CONTEXT': {
 141             'client': {
 142                 'clientName': 'ANDROID_MUSIC',
 143                 'clientVersion': '4.57',
 144             }
 145         },
 146         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 147         'REQUIRE_JS_PLAYER': False
 148     },
 149     'android_creator': {
 150         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 151         'INNERTUBE_CONTEXT': {
 152             'client': {
 153                 'clientName': 'ANDROID_CREATOR',
 154                 'clientVersion': '21.47',
 155             },
 156         },
 157         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 158         'REQUIRE_JS_PLAYER': False
 159     },
 160     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 161     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 162     'ios': {
 163         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 164         'INNERTUBE_CONTEXT': {
 165             'client': {
 166                 'clientName': 'IOS',
 167                 'clientVersion': '16.46',
 168                 'deviceModel': 'iPhone14,3',
 169             }
 170         },
 171         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 172         'REQUIRE_JS_PLAYER': False
 173     },
 174     'ios_embedded': {
 175         'INNERTUBE_CONTEXT': {
 176             'client': {
 177                 'clientName': 'IOS_MESSAGES_EXTENSION',
 178                 'clientVersion': '16.46',
 179                 'deviceModel': 'iPhone14,3',
 180             },
 181         },
 182         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 183         'REQUIRE_JS_PLAYER': False
 184     },
 185     'ios_music': {
 186         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 187         'INNERTUBE_CONTEXT': {
 188             'client': {
 189                 'clientName': 'IOS_MUSIC',
 190                 'clientVersion': '4.57',
 191             },
 192         },
 193         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 194         'REQUIRE_JS_PLAYER': False
 195     },
 196     'ios_creator': {
 197         'INNERTUBE_CONTEXT': {
 198             'client': {
 199                 'clientName': 'IOS_CREATOR',
 200                 'clientVersion': '21.47',
 201             },
 202         },
 203         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 204         'REQUIRE_JS_PLAYER': False
 205     },
 206     # mweb has 'ultralow' formats
 207     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 208     'mweb': {
 209         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 210         'INNERTUBE_CONTEXT': {
 211             'client': {
 212                 'clientName': 'MWEB',
 213                 'clientVersion': '2.20211221.01.00',
 214             }
 215         },
 216         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 217     },
 218     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 219     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 220     'tv_embedded': {
 221         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 222         'INNERTUBE_CONTEXT': {
 223             'client': {
 224                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 225                 'clientVersion': '2.0',
 226             },
 227         },
 228         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 229     },
 230 }
 231
 232
 233 def _split_innertube_client(client_name):
 234     variant, *base = client_name.rsplit('.', 1)
 235     if base:
 236         return variant, base[0], variant
 237     base, *variant = client_name.split('_', 1)
 238     return client_name, base, variant[0] if variant else None
 239
 240
 241 def build_innertube_clients():
 242     THIRD_PARTY = {
 243         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 244     }
 245     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 246     priority = qualities(BASE_CLIENTS[::-1])
 247
 248     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 249         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 250         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 251         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 252         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 253
 254         _, base_client, variant = _split_innertube_client(client)
 255         ytcfg['priority'] = 10 * priority(base_client)
 256
 257         if not variant:
 258             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 259             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 260             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 261             embedscreen['priority'] -= 3
 262         elif variant == 'embedded':
 263             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 264             ytcfg['priority'] -= 2
 265         else:
 266             ytcfg['priority'] -= 3
 267
 268
 269 build_innertube_clients()
 270
 271
 272 class YoutubeBaseInfoExtractor(InfoExtractor):
 273     """Provide base functions for Youtube extractors"""
 274
 275     _RESERVED_NAMES = (
 276         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 277         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 278         r'browse|oembed|get_video_info|iframe_api|s/player|'
 279         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 280
 281     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 282
 283     # _NETRC_MACHINE = 'youtube'
 284
 285     # If True it will raise an error if no login info is provided
 286     _LOGIN_REQUIRED = False
 287
 288     _INVIDIOUS_SITES = (
 289         # invidious-redirect websites
 290         r'(?:www\.)?redirect\.invidious\.io',
 291         r'(?:(?:www|dev)\.)?invidio\.us',
 292         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 293         r'(?:www\.)?invidious\.pussthecat\.org',
 294         r'(?:www\.)?invidious\.zee\.li',
 295         r'(?:www\.)?invidious\.ethibox\.fr',
 296         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 297         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 298         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 299         # youtube-dl invidious instances list
 300         r'(?:(?:www|no)\.)?invidiou\.sh',
 301         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 302         r'(?:www\.)?invidious\.kabi\.tk',
 303         r'(?:www\.)?invidious\.mastodon\.host',
 304         r'(?:www\.)?invidious\.zapashcanon\.fr',
 305         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 306         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 307         r'(?:www\.)?invidious\.himiko\.cloud',
 308         r'(?:www\.)?invidious\.reallyancient\.tech',
 309         r'(?:www\.)?invidious\.tube',
 310         r'(?:www\.)?invidiou\.site',
 311         r'(?:www\.)?invidious\.site',
 312         r'(?:www\.)?invidious\.xyz',
 313         r'(?:www\.)?invidious\.nixnet\.xyz',
 314         r'(?:www\.)?invidious\.048596\.xyz',
 315         r'(?:www\.)?invidious\.drycat\.fr',
 316         r'(?:www\.)?inv\.skyn3t\.in',
 317         r'(?:www\.)?tube\.poal\.co',
 318         r'(?:www\.)?tube\.connect\.cafe',
 319         r'(?:www\.)?vid\.wxzm\.sx',
 320         r'(?:www\.)?vid\.mint\.lgbt',
 321         r'(?:www\.)?vid\.puffyan\.us',
 322         r'(?:www\.)?yewtu\.be',
 323         r'(?:www\.)?yt\.elukerio\.org',
 324         r'(?:www\.)?yt\.lelux\.fi',
 325         r'(?:www\.)?invidious\.ggc-project\.de',
 326         r'(?:www\.)?yt\.maisputain\.ovh',
 327         r'(?:www\.)?ytprivate\.com',
 328         r'(?:www\.)?invidious\.13ad\.de',
 329         r'(?:www\.)?invidious\.toot\.koeln',
 330         r'(?:www\.)?invidious\.fdn\.fr',
 331         r'(?:www\.)?watch\.nettohikari\.com',
 332         r'(?:www\.)?invidious\.namazso\.eu',
 333         r'(?:www\.)?invidious\.silkky\.cloud',
 334         r'(?:www\.)?invidious\.exonip\.de',
 335         r'(?:www\.)?invidious\.riverside\.rocks',
 336         r'(?:www\.)?invidious\.blamefran\.net',
 337         r'(?:www\.)?invidious\.moomoo\.de',
 338         r'(?:www\.)?ytb\.trom\.tf',
 339         r'(?:www\.)?yt\.cyberhost\.uk',
 340         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 341         r'(?:www\.)?qklhadlycap4cnod\.onion',
 342         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 343         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 344         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 345         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 346         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 347         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 348         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 349         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 350         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 351         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 352         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 353         r'(?:www\.)?piped\.kavin\.rocks',
 354         r'(?:www\.)?piped\.silkky\.cloud',
 355         r'(?:www\.)?piped\.tokhmi\.xyz',
 356         r'(?:www\.)?piped\.moomoo\.me',
 357         r'(?:www\.)?il\.ax',
 358         r'(?:www\.)?piped\.syncpundit\.com',
 359         r'(?:www\.)?piped\.mha\.fi',
 360         r'(?:www\.)?piped\.mint\.lgbt',
 361         r'(?:www\.)?piped\.privacy\.com\.de',
 362     )
 363
 364     def _initialize_consent(self):
 365         cookies = self._get_cookies('https://www.youtube.com/')
 366         if cookies.get('__Secure-3PSID'):
 367             return
 368         consent_id = None
 369         consent = cookies.get('CONSENT')
 370         if consent:
 371             if 'YES' in consent.value:
 372                 return
 373             consent_id = self._search_regex(
 374                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 375         if not consent_id:
 376             consent_id = random.randint(100, 999)
 377         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 378
 379     def _initialize_pref(self):
 380         cookies = self._get_cookies('https://www.youtube.com/')
 381         pref_cookie = cookies.get('PREF')
 382         pref = {}
 383         if pref_cookie:
 384             try:
 385                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 386             except ValueError:
 387                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 388         pref.update({'hl': 'en', 'tz': 'UTC'})
 389         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 390
 391     def _real_initialize(self):
 392         self._initialize_pref()
 393         self._initialize_consent()
 394         self._check_login_required()
 395
 396     def _check_login_required(self):
 397         if self._LOGIN_REQUIRED and not self._cookies_passed:
 398             self.raise_login_required('Login details are needed to download this content', method='cookies')
 399
 400     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 401     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 402     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 403
 404     def _get_default_ytcfg(self, client='web'):
 405         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 406
 407     def _get_innertube_host(self, client='web'):
 408         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 409
 410     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 411         # try_get but with fallback to default ytcfg client values when present
 412         _func = lambda y: try_get(y, getter, expected_type)
 413         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 414
 415     def _extract_client_name(self, ytcfg, default_client='web'):
 416         return self._ytcfg_get_safe(
 417             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 418                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 419
 420     def _extract_client_version(self, ytcfg, default_client='web'):
 421         return self._ytcfg_get_safe(
 422             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 423                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 424
 425     def _extract_api_key(self, ytcfg=None, default_client='web'):
 426         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 427
 428     def _extract_context(self, ytcfg=None, default_client='web'):
 429         context = get_first(
 430             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 431         # Enforce language and tz for extraction
 432         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 433         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 434         return context
 435
 436     _SAPISID = None
 437
 438     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 439         time_now = round(time.time())
 440         if self._SAPISID is None:
 441             yt_cookies = self._get_cookies('https://www.youtube.com')
 442             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 443             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 444             sapisid_cookie = dict_get(
 445                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 446             if sapisid_cookie and sapisid_cookie.value:
 447                 self._SAPISID = sapisid_cookie.value
 448                 self.write_debug('Extracted SAPISID cookie')
 449                 # SAPISID cookie is required if not already present
 450                 if not yt_cookies.get('SAPISID'):
 451                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 452                     self._set_cookie(
 453                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 454             else:
 455                 self._SAPISID = False
 456         if not self._SAPISID:
 457             return None
 458         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 459         sapisidhash = hashlib.sha1(
 460             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 461         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 462
 463     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 464                   note='Downloading API JSON', errnote='Unable to download API page',
 465                   context=None, api_key=None, api_hostname=None, default_client='web'):
 466
 467         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 468         data.update(query)
 469         real_headers = self.generate_api_headers(default_client=default_client)
 470         real_headers.update({'content-type': 'application/json'})
 471         if headers:
 472             real_headers.update(headers)
 473         return self._download_json(
 474             f'https://{api_hostname or self._get_innertube_host(default_client)}/youtubei/v1/{ep}',
 475             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 476             data=json.dumps(data).encode('utf8'), headers=real_headers,
 477             query={'key': api_key or self._extract_api_key(), 'prettyPrint': 'false'})
 478
 479     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 480         data = self._search_regex(
 481             (fr'{self._YT_INITIAL_DATA_RE}\s*{self._YT_INITIAL_BOUNDARY_RE}',
 482              self._YT_INITIAL_DATA_RE), webpage, 'yt initial data', fatal=fatal)
 483         if data:
 484             return self._parse_json(data, item_id, fatal=fatal)
 485
 486     @staticmethod
 487     def _extract_session_index(*data):
 488         """
 489         Index of current account in account list.
 490         See: https://github.com/yt-dlp/yt-dlp/pull/519
 491         """
 492         for ytcfg in data:
 493             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 494             if session_index is not None:
 495                 return session_index
 496
 497     # Deprecated?
 498     def _extract_identity_token(self, ytcfg=None, webpage=None):
 499         if ytcfg:
 500             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 501             if token:
 502                 return token
 503         if webpage:
 504             return self._search_regex(
 505                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 506                 'identity token', default=None, fatal=False)
 507
 508     @staticmethod
 509     def _extract_account_syncid(*args):
 510         """
 511         Extract syncId required to download private playlists of secondary channels
 512         @params response and/or ytcfg
 513         """
 514         for data in args:
 515             # ytcfg includes channel_syncid if on secondary channel
 516             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 517             if delegated_sid:
 518                 return delegated_sid
 519             sync_ids = (try_get(
 520                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 521                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 522             if len(sync_ids) >= 2 and sync_ids[1]:
 523                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 524                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 525                 return sync_ids[0]
 526
 527     @staticmethod
 528     def _extract_visitor_data(*args):
 529         """
 530         Extracts visitorData from an API response or ytcfg
 531         Appears to be used to track session state
 532         """
 533         return get_first(
 534             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 535             expected_type=str)
 536
 537     @functools.cached_property
 538     def is_authenticated(self):
 539         return bool(self._generate_sapisidhash_header())
 540
 541     def extract_ytcfg(self, video_id, webpage):
 542         if not webpage:
 543             return {}
 544         return self._parse_json(
 545             self._search_regex(
 546                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 547                 default='{}'), video_id, fatal=False) or {}
 548
 549     def generate_api_headers(
 550             self, *, ytcfg=None, account_syncid=None, session_index=None,
 551             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 552
 553         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
 554         headers = {
 555             'X-YouTube-Client-Name': compat_str(
 556                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 557             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 558             'Origin': origin,
 559             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 560             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 561             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 562         }
 563         if session_index is None:
 564             session_index = self._extract_session_index(ytcfg)
 565         if account_syncid or session_index is not None:
 566             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 567
 568         auth = self._generate_sapisidhash_header(origin)
 569         if auth is not None:
 570             headers['Authorization'] = auth
 571             headers['X-Origin'] = origin
 572         return {h: v for h, v in headers.items() if v is not None}
 573
 574     def _download_ytcfg(self, client, video_id):
 575         url = {
 576             'web': 'https://www.youtube.com',
 577             'web_music': 'https://music.youtube.com',
 578             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 579         }.get(client)
 580         if not url:
 581             return {}
 582         webpage = self._download_webpage(
 583             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 584         return self.extract_ytcfg(video_id, webpage) or {}
 585
 586     @staticmethod
 587     def _build_api_continuation_query(continuation, ctp=None):
 588         query = {
 589             'continuation': continuation
 590         }
 591         # TODO: Inconsistency with clickTrackingParams.
 592         # Currently we have a fixed ctp contained within context (from ytcfg)
 593         # and a ctp in root query for continuation.
 594         if ctp:
 595             query['clickTracking'] = {'clickTrackingParams': ctp}
 596         return query
 597
 598     @classmethod
 599     def _extract_next_continuation_data(cls, renderer):
 600         next_continuation = try_get(
 601             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 602                        lambda x: x['continuation']['reloadContinuationData']), dict)
 603         if not next_continuation:
 604             return
 605         continuation = next_continuation.get('continuation')
 606         if not continuation:
 607             return
 608         ctp = next_continuation.get('clickTrackingParams')
 609         return cls._build_api_continuation_query(continuation, ctp)
 610
 611     @classmethod
 612     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 613         if isinstance(continuation_ep, dict):
 614             continuation = try_get(
 615                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 616             if not continuation:
 617                 return
 618             ctp = continuation_ep.get('clickTrackingParams')
 619             return cls._build_api_continuation_query(continuation, ctp)
 620
 621     @classmethod
 622     def _extract_continuation(cls, renderer):
 623         next_continuation = cls._extract_next_continuation_data(renderer)
 624         if next_continuation:
 625             return next_continuation
 626
 627         contents = []
 628         for key in ('contents', 'items'):
 629             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 630
 631         for content in contents:
 632             if not isinstance(content, dict):
 633                 continue
 634             continuation_ep = try_get(
 635                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 636                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 637                 dict)
 638             continuation = cls._extract_continuation_ep_data(continuation_ep)
 639             if continuation:
 640                 return continuation
 641
 642     @classmethod
 643     def _extract_alerts(cls, data):
 644         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 645             if not isinstance(alert_dict, dict):
 646                 continue
 647             for alert in alert_dict.values():
 648                 alert_type = alert.get('type')
 649                 if not alert_type:
 650                     continue
 651                 message = cls._get_text(alert, 'text')
 652                 if message:
 653                     yield alert_type, message
 654
 655     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 656         errors = []
 657         warnings = []
 658         for alert_type, alert_message in alerts:
 659             if alert_type.lower() == 'error' and fatal:
 660                 errors.append([alert_type, alert_message])
 661             else:
 662                 warnings.append([alert_type, alert_message])
 663
 664         for alert_type, alert_message in (warnings + errors[:-1]):
 665             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 666         if errors:
 667             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 668
 669     def _extract_and_report_alerts(self, data, *args, **kwargs):
 670         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 671
 672     def _extract_badges(self, renderer: dict):
 673         badges = set()
 674         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 675             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 676             if label:
 677                 badges.add(label.lower())
 678         return badges
 679
 680     @staticmethod
 681     def _get_text(data, *path_list, max_runs=None):
 682         for path in path_list or [None]:
 683             if path is None:
 684                 obj = [data]
 685             else:
 686                 obj = traverse_obj(data, path, default=[])
 687                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 688                     obj = [obj]
 689             for item in obj:
 690                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 691                 if text:
 692                     return text
 693                 runs = try_get(item, lambda x: x['runs'], list) or []
 694                 if not runs and isinstance(item, list):
 695                     runs = item
 696
 697                 runs = runs[:min(len(runs), max_runs or len(runs))]
 698                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 699                 if text:
 700                     return text
 701
 702     def _get_count(self, data, *path_list):
 703         count_text = self._get_text(data, *path_list) or ''
 704         count = parse_count(count_text)
 705         if count is None:
 706             count = str_to_int(
 707                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 708         return count
 709
 710     @staticmethod
 711     def _extract_thumbnails(data, *path_list):
 712         """
 713         Extract thumbnails from thumbnails dict
 714         @param path_list: path list to level that contains 'thumbnails' key
 715         """
 716         thumbnails = []
 717         for path in path_list or [()]:
 718             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 719                 thumbnail_url = url_or_none(thumbnail.get('url'))
 720                 if not thumbnail_url:
 721                     continue
 722                 # Sometimes youtube gives a wrong thumbnail URL. See:
 723                 # https://github.com/yt-dlp/yt-dlp/issues/233
 724                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 725                 if 'maxresdefault' in thumbnail_url:
 726                     thumbnail_url = thumbnail_url.split('?')[0]
 727                 thumbnails.append({
 728                     'url': thumbnail_url,
 729                     'height': int_or_none(thumbnail.get('height')),
 730                     'width': int_or_none(thumbnail.get('width')),
 731                 })
 732         return thumbnails
 733
 734     @staticmethod
 735     def extract_relative_time(relative_time_text):
 736         """
 737         Extracts a relative time from string and converts to dt object
 738         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 739         """
 740         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 741         if mobj:
 742             start = mobj.group('start')
 743             if start:
 744                 return datetime_from_str(start)
 745             try:
 746                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 747             except ValueError:
 748                 return None
 749
 750     def _extract_time_text(self, renderer, *path_list):
 751         """@returns (timestamp, time_text)"""
 752         text = self._get_text(renderer, *path_list) or ''
 753         dt = self.extract_relative_time(text)
 754         timestamp = None
 755         if isinstance(dt, datetime.datetime):
 756             timestamp = calendar.timegm(dt.timetuple())
 757
 758         if timestamp is None:
 759             timestamp = (
 760                 unified_timestamp(text) or unified_timestamp(
 761                     self._search_regex(
 762                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 763                         text.lower(), 'time text', default=None)))
 764
 765         if text and timestamp is None:
 766             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 767         return timestamp, text
 768
 769     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 770                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 771                           default_client='web'):
 772         response = None
 773         last_error = None
 774         count = -1
 775         retries = self.get_param('extractor_retries', 3)
 776         if check_get_keys is None:
 777             check_get_keys = []
 778         while count < retries:
 779             count += 1
 780             if last_error:
 781                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 782             try:
 783                 response = self._call_api(
 784                     ep=ep, fatal=True, headers=headers,
 785                     video_id=item_id, query=query,
 786                     context=self._extract_context(ytcfg, default_client),
 787                     api_key=self._extract_api_key(ytcfg, default_client),
 788                     api_hostname=api_hostname, default_client=default_client,
 789                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 790             except ExtractorError as e:
 791                 if isinstance(e.cause, network_exceptions):
 792                     if isinstance(e.cause, compat_HTTPError):
 793                         first_bytes = e.cause.read(512)
 794                         if not is_html(first_bytes):
 795                             yt_error = try_get(
 796                                 self._parse_json(
 797                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 798                                 lambda x: x['error']['message'], compat_str)
 799                             if yt_error:
 800                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 801                     # Downloading page may result in intermittent 5xx HTTP error
 802                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 803                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 804                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 805                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 806                         last_error = error_to_compat_str(e.cause or e.msg)
 807                         if count < retries:
 808                             continue
 809                 if fatal:
 810                     raise
 811                 else:
 812                     self.report_warning(error_to_compat_str(e))
 813                     return
 814
 815             else:
 816                 try:
 817                     self._extract_and_report_alerts(response, only_once=True)
 818                 except ExtractorError as e:
 819                     # YouTube servers may return errors we want to retry on in a 200 OK response
 820                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 821                     if 'unknown error' in e.msg.lower():
 822                         last_error = e.msg
 823                         continue
 824                     if fatal:
 825                         raise
 826                     self.report_warning(error_to_compat_str(e))
 827                     return
 828                 if not check_get_keys or dict_get(response, check_get_keys):
 829                     break
 830                 # Youtube sometimes sends incomplete data
 831                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 832                 last_error = 'Incomplete data received'
 833                 if count >= retries:
 834                     if fatal:
 835                         raise ExtractorError(last_error)
 836                     else:
 837                         self.report_warning(last_error)
 838                         return
 839         return response
 840
 841     @staticmethod
 842     def is_music_url(url):
 843         return re.match(r'https?://music\.youtube\.com/', url) is not None
 844
 845     def _extract_video(self, renderer):
 846         video_id = renderer.get('videoId')
 847         title = self._get_text(renderer, 'title')
 848         description = self._get_text(renderer, 'descriptionSnippet')
 849         duration = parse_duration(self._get_text(
 850             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 851         if duration is None:
 852             duration = parse_duration(self._search_regex(
 853                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 854                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 855                 video_id, default=None, group='duration'))
 856
 857         view_count = self._get_count(renderer, 'viewCountText')
 858
 859         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 860         channel_id = traverse_obj(
 861             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 862             expected_type=str, get_all=False)
 863         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 864         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 865         overlay_style = traverse_obj(
 866             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 867             get_all=False, expected_type=str)
 868         badges = self._extract_badges(renderer)
 869         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 870         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 871             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 872             expected_type=str)) or ''
 873         url = f'https://www.youtube.com/watch?v={video_id}'
 874         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 875             url = f'https://www.youtube.com/shorts/{video_id}'
 876
 877         return {
 878             '_type': 'url',
 879             'ie_key': YoutubeIE.ie_key(),
 880             'id': video_id,
 881             'url': url,
 882             'title': title,
 883             'description': description,
 884             'duration': duration,
 885             'view_count': view_count,
 886             'uploader': uploader,
 887             'channel_id': channel_id,
 888             'thumbnails': thumbnails,
 889             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 890                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 891                             else None),
 892             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 893                             else 'was_live' if 'streamed' in time_text.lower()
 894                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 895                             else None),
 896             'release_timestamp': scheduled_timestamp,
 897             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 898         }
 899
 900
 901 class YoutubeIE(YoutubeBaseInfoExtractor):
 902     IE_DESC = 'YouTube'
 903     _VALID_URL = r"""(?x)^
 904                      (
 905                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 906                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 907                             (?:www\.)?deturl\.com/www\.youtube\.com|
 908                             (?:www\.)?pwnyoutube\.com|
 909                             (?:www\.)?hooktube\.com|
 910                             (?:www\.)?yourepeat\.com|
 911                             tube\.majestyc\.net|
 912                             %(invidious)s|
 913                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 914                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 915                          (?:                                                  # the various things that can precede the ID:
 916                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 917                              |(?:                                             # or the v= param in all its forms
 918                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 919                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 920                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 921                                  v=
 922                              )
 923                          ))
 924                          |(?:
 925                             youtu\.be|                                        # just youtu.be/xxxx
 926                             vid\.plus|                                        # or vid.plus/xxxx
 927                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 928                             %(invidious)s
 929                          )/
 930                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 931                          )
 932                      )?                                                       # all until now is optional -> you can pass the naked ID
 933                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 934                      (?(1).+)?                                                # if we found the ID, everything can follow
 935                      (?:\#|$)""" % {
 936         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 937     }
 938     _PLAYER_INFO_RE = (
 939         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 940         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 941         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 942     )
 943     _formats = {
 944         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 945         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 946         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 947         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 948         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 949         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 950         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 951         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 953         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 954         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 955         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 956         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 957         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 958         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 959         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 960         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 961         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 962
 963
 964         # 3D videos
 965         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 966         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 967         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 968         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 969         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 970         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 971         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 972
 973         # Apple HTTP Live Streaming
 974         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 975         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 976         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 977         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 978         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 979         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 980         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 981         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 982
 983         # DASH mp4 video
 984         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 985         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 986         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 988         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 989         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 990         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 991         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 992         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 993         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 994         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 995         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 996
 997         # Dash mp4 audio
 998         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 999         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1000         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1001         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1002         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1003         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1004         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1005
1006         # Dash webm
1007         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1008         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1009         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1010         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1011         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1012         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1013         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1014         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1015         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1016         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1017         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1018         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1020         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1021         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1022         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1023         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1024         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1025         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1026         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1027         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1028         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1029
1030         # Dash webm audio
1031         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1032         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1033
1034         # Dash webm audio with opus inside
1035         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1036         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1037         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1038
1039         # RTMP (unnamed)
1040         '_rtmp': {'protocol': 'rtmp'},
1041
1042         # av01 video only formats sometimes served with "unknown" codecs
1043         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1044         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1045         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1046         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1047         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1048         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1049         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1050         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1051     }
1052     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1053
1054     _GEO_BYPASS = False
1055
1056     IE_NAME = 'youtube'
1057     _TESTS = [
1058         {
1059             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1060             'info_dict': {
1061                 'id': 'BaW_jenozKc',
1062                 'ext': 'mp4',
1063                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1064                 'uploader': 'Philipp Hagemeister',
1065                 'uploader_id': 'phihag',
1066                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1067                 'channel': 'Philipp Hagemeister',
1068                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1069                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1070                 'upload_date': '20121002',
1071                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1072                 'categories': ['Science & Technology'],
1073                 'tags': ['youtube-dl'],
1074                 'duration': 10,
1075                 'view_count': int,
1076                 'like_count': int,
1077                 'availability': 'public',
1078                 'playable_in_embed': True,
1079                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1080                 'live_status': 'not_live',
1081                 'age_limit': 0,
1082                 'start_time': 1,
1083                 'end_time': 9,
1084                 'channel_follower_count': int
1085             }
1086         },
1087         {
1088             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1089             'note': 'Embed-only video (#1746)',
1090             'info_dict': {
1091                 'id': 'yZIXLfi8CZQ',
1092                 'ext': 'mp4',
1093                 'upload_date': '20120608',
1094                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1095                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1096                 'uploader': 'SET India',
1097                 'uploader_id': 'setindia',
1098                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1099                 'age_limit': 18,
1100             },
1101             'skip': 'Private video',
1102         },
1103         {
1104             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1105             'note': 'Use the first video ID in the URL',
1106             'info_dict': {
1107                 'id': 'BaW_jenozKc',
1108                 'ext': 'mp4',
1109                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1110                 'uploader': 'Philipp Hagemeister',
1111                 'uploader_id': 'phihag',
1112                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1113                 'channel': 'Philipp Hagemeister',
1114                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1115                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1116                 'upload_date': '20121002',
1117                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1118                 'categories': ['Science & Technology'],
1119                 'tags': ['youtube-dl'],
1120                 'duration': 10,
1121                 'view_count': int,
1122                 'like_count': int,
1123                 'availability': 'public',
1124                 'playable_in_embed': True,
1125                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1126                 'live_status': 'not_live',
1127                 'age_limit': 0,
1128                 'channel_follower_count': int
1129             },
1130             'params': {
1131                 'skip_download': True,
1132             },
1133         },
1134         {
1135             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1136             'note': '256k DASH audio (format 141) via DASH manifest',
1137             'info_dict': {
1138                 'id': 'a9LDPn-MO4I',
1139                 'ext': 'm4a',
1140                 'upload_date': '20121002',
1141                 'uploader_id': '8KVIDEO',
1142                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1143                 'description': '',
1144                 'uploader': '8KVIDEO',
1145                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1146             },
1147             'params': {
1148                 'youtube_include_dash_manifest': True,
1149                 'format': '141',
1150             },
1151             'skip': 'format 141 not served anymore',
1152         },
1153         # DASH manifest with encrypted signature
1154         {
1155             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1156             'info_dict': {
1157                 'id': 'IB3lcPjvWLA',
1158                 'ext': 'm4a',
1159                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1160                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1161                 'duration': 244,
1162                 'uploader': 'AfrojackVEVO',
1163                 'uploader_id': 'AfrojackVEVO',
1164                 'upload_date': '20131011',
1165                 'abr': 129.495,
1166                 'like_count': int,
1167                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1168                 'playable_in_embed': True,
1169                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1170                 'view_count': int,
1171                 'track': 'The Spark',
1172                 'live_status': 'not_live',
1173                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1174                 'channel': 'Afrojack',
1175                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1176                 'tags': 'count:19',
1177                 'availability': 'public',
1178                 'categories': ['Music'],
1179                 'age_limit': 0,
1180                 'alt_title': 'The Spark',
1181                 'channel_follower_count': int
1182             },
1183             'params': {
1184                 'youtube_include_dash_manifest': True,
1185                 'format': '141/bestaudio[ext=m4a]',
1186             },
1187         },
1188         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1189         {
1190             'note': 'Embed allowed age-gate video',
1191             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1192             'info_dict': {
1193                 'id': 'HtVdAasjOgU',
1194                 'ext': 'mp4',
1195                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1196                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1197                 'duration': 142,
1198                 'uploader': 'The Witcher',
1199                 'uploader_id': 'WitcherGame',
1200                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1201                 'upload_date': '20140605',
1202                 'age_limit': 18,
1203                 'categories': ['Gaming'],
1204                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1205                 'availability': 'needs_auth',
1206                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1207                 'like_count': int,
1208                 'channel': 'The Witcher',
1209                 'live_status': 'not_live',
1210                 'tags': 'count:17',
1211                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1212                 'playable_in_embed': True,
1213                 'view_count': int,
1214                 'channel_follower_count': int
1215             },
1216         },
1217         {
1218             'note': 'Age-gate video with embed allowed in public site',
1219             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1220             'info_dict': {
1221                 'id': 'HsUATh_Nc2U',
1222                 'ext': 'mp4',
1223                 'title': 'Godzilla 2 (Official Video)',
1224                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1225                 'upload_date': '20200408',
1226                 'uploader_id': 'FlyingKitty900',
1227                 'uploader': 'FlyingKitty',
1228                 'age_limit': 18,
1229                 'availability': 'needs_auth',
1230                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1231                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1232                 'channel': 'FlyingKitty',
1233                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1234                 'view_count': int,
1235                 'categories': ['Entertainment'],
1236                 'live_status': 'not_live',
1237                 'tags': ['Flyingkitty', 'godzilla 2'],
1238                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1239                 'like_count': int,
1240                 'duration': 177,
1241                 'playable_in_embed': True,
1242                 'channel_follower_count': int
1243             },
1244         },
1245         {
1246             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1247             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1248             'info_dict': {
1249                 'id': 'Tq92D6wQ1mg',
1250                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1251                 'ext': 'mp4',
1252                 'upload_date': '20191228',
1253                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1254                 'uploader': 'Projekt Melody',
1255                 'description': 'md5:17eccca93a786d51bc67646756894066',
1256                 'age_limit': 18,
1257                 'like_count': int,
1258                 'availability': 'needs_auth',
1259                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1260                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1261                 'view_count': int,
1262                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1263                 'channel': 'Projekt Melody',
1264                 'live_status': 'not_live',
1265                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1266                 'playable_in_embed': True,
1267                 'categories': ['Entertainment'],
1268                 'duration': 106,
1269                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1270                 'channel_follower_count': int
1271             },
1272         },
1273         {
1274             'note': 'Non-Agegated non-embeddable video',
1275             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1276             'info_dict': {
1277                 'id': 'MeJVWBSsPAY',
1278                 'ext': 'mp4',
1279                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1280                 'uploader': 'Herr Lurik',
1281                 'uploader_id': 'st3in234',
1282                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1283                 'upload_date': '20130730',
1284                 'track': 'Such mich find mich',
1285                 'age_limit': 0,
1286                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1287                 'like_count': int,
1288                 'playable_in_embed': False,
1289                 'creator': 'OOMPH!',
1290                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1291                 'view_count': int,
1292                 'alt_title': 'Such mich find mich',
1293                 'duration': 210,
1294                 'channel': 'Herr Lurik',
1295                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1296                 'categories': ['Music'],
1297                 'availability': 'public',
1298                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1299                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1300                 'live_status': 'not_live',
1301                 'artist': 'OOMPH!',
1302                 'channel_follower_count': int
1303             },
1304         },
1305         {
1306             'note': 'Non-bypassable age-gated video',
1307             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1308             'only_matching': True,
1309         },
1310         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1311         # YouTube Red ad is not captured for creator
1312         {
1313             'url': '__2ABJjxzNo',
1314             'info_dict': {
1315                 'id': '__2ABJjxzNo',
1316                 'ext': 'mp4',
1317                 'duration': 266,
1318                 'upload_date': '20100430',
1319                 'uploader_id': 'deadmau5',
1320                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1321                 'creator': 'deadmau5',
1322                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1323                 'uploader': 'deadmau5',
1324                 'title': 'Deadmau5 - Some Chords (HD)',
1325                 'alt_title': 'Some Chords',
1326                 'availability': 'public',
1327                 'tags': 'count:14',
1328                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1329                 'view_count': int,
1330                 'live_status': 'not_live',
1331                 'channel': 'deadmau5',
1332                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1333                 'like_count': int,
1334                 'track': 'Some Chords',
1335                 'artist': 'deadmau5',
1336                 'playable_in_embed': True,
1337                 'age_limit': 0,
1338                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1339                 'categories': ['Music'],
1340                 'album': 'Some Chords',
1341                 'channel_follower_count': int
1342             },
1343             'expected_warnings': [
1344                 'DASH manifest missing',
1345             ]
1346         },
1347         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1348         {
1349             'url': 'lqQg6PlCWgI',
1350             'info_dict': {
1351                 'id': 'lqQg6PlCWgI',
1352                 'ext': 'mp4',
1353                 'duration': 6085,
1354                 'upload_date': '20150827',
1355                 'uploader_id': 'olympic',
1356                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1357                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1358                 'uploader': 'Olympics',
1359                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1360                 'like_count': int,
1361                 'release_timestamp': 1343767800,
1362                 'playable_in_embed': True,
1363                 'categories': ['Sports'],
1364                 'release_date': '20120731',
1365                 'channel': 'Olympics',
1366                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1367                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1368                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1369                 'age_limit': 0,
1370                 'availability': 'public',
1371                 'live_status': 'was_live',
1372                 'view_count': int,
1373                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1374                 'channel_follower_count': int
1375             },
1376             'params': {
1377                 'skip_download': 'requires avconv',
1378             }
1379         },
1380         # Non-square pixels
1381         {
1382             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1383             'info_dict': {
1384                 'id': '_b-2C3KPAM0',
1385                 'ext': 'mp4',
1386                 'stretched_ratio': 16 / 9.,
1387                 'duration': 85,
1388                 'upload_date': '20110310',
1389                 'uploader_id': 'AllenMeow',
1390                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1391                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1392                 'uploader': '孫ᄋᄅ',
1393                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1394                 'playable_in_embed': True,
1395                 'channel': '孫ᄋᄅ',
1396                 'age_limit': 0,
1397                 'tags': 'count:11',
1398                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1399                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1400                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1401                 'view_count': int,
1402                 'categories': ['People & Blogs'],
1403                 'like_count': int,
1404                 'live_status': 'not_live',
1405                 'availability': 'unlisted',
1406                 'channel_follower_count': int
1407             },
1408         },
1409         # url_encoded_fmt_stream_map is empty string
1410         {
1411             'url': 'qEJwOuvDf7I',
1412             'info_dict': {
1413                 'id': 'qEJwOuvDf7I',
1414                 'ext': 'webm',
1415                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1416                 'description': '',
1417                 'upload_date': '20150404',
1418                 'uploader_id': 'spbelect',
1419                 'uploader': 'Наблюдатели Петербурга',
1420             },
1421             'params': {
1422                 'skip_download': 'requires avconv',
1423             },
1424             'skip': 'This live event has ended.',
1425         },
1426         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1427         {
1428             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1429             'info_dict': {
1430                 'id': 'FIl7x6_3R5Y',
1431                 'ext': 'webm',
1432                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1433                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1434                 'duration': 220,
1435                 'upload_date': '20150625',
1436                 'uploader_id': 'dorappi2000',
1437                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1438                 'uploader': 'dorappi2000',
1439                 'formats': 'mincount:31',
1440             },
1441             'skip': 'not actual anymore',
1442         },
1443         # DASH manifest with segment_list
1444         {
1445             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1446             'md5': '8ce563a1d667b599d21064e982ab9e31',
1447             'info_dict': {
1448                 'id': 'CsmdDsKjzN8',
1449                 'ext': 'mp4',
1450                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1451                 'uploader': 'Airtek',
1452                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1453                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1454                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1455             },
1456             'params': {
1457                 'youtube_include_dash_manifest': True,
1458                 'format': '135',  # bestvideo
1459             },
1460             'skip': 'This live event has ended.',
1461         },
1462         {
1463             # Multifeed videos (multiple cameras), URL is for Main Camera
1464             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1465             'info_dict': {
1466                 'id': 'jvGDaLqkpTg',
1467                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1468                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1469             },
1470             'playlist': [{
1471                 'info_dict': {
1472                     'id': 'jvGDaLqkpTg',
1473                     'ext': 'mp4',
1474                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1475                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1476                     'duration': 10643,
1477                     'upload_date': '20161111',
1478                     'uploader': 'Team PGP',
1479                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1480                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1481                 },
1482             }, {
1483                 'info_dict': {
1484                     'id': '3AKt1R1aDnw',
1485                     'ext': 'mp4',
1486                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1487                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1488                     'duration': 10991,
1489                     'upload_date': '20161111',
1490                     'uploader': 'Team PGP',
1491                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1492                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1493                 },
1494             }, {
1495                 'info_dict': {
1496                     'id': 'RtAMM00gpVc',
1497                     'ext': 'mp4',
1498                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1499                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1500                     'duration': 10995,
1501                     'upload_date': '20161111',
1502                     'uploader': 'Team PGP',
1503                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1504                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1505                 },
1506             }, {
1507                 'info_dict': {
1508                     'id': '6N2fdlP3C5U',
1509                     'ext': 'mp4',
1510                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1511                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1512                     'duration': 10990,
1513                     'upload_date': '20161111',
1514                     'uploader': 'Team PGP',
1515                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1516                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1517                 },
1518             }],
1519             'params': {
1520                 'skip_download': True,
1521             },
1522             'skip': 'Not multifeed anymore',
1523         },
1524         {
1525             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1526             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1527             'info_dict': {
1528                 'id': 'gVfLd0zydlo',
1529                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1530             },
1531             'playlist_count': 2,
1532             'skip': 'Not multifeed anymore',
1533         },
1534         {
1535             'url': 'https://vid.plus/FlRa-iH7PGw',
1536             'only_matching': True,
1537         },
1538         {
1539             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1540             'only_matching': True,
1541         },
1542         {
1543             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1544             # Also tests cut-off URL expansion in video description (see
1545             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1546             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1547             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1548             'info_dict': {
1549                 'id': 'lsguqyKfVQg',
1550                 'ext': 'mp4',
1551                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1552                 'alt_title': 'Dark Walk',
1553                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1554                 'duration': 133,
1555                 'upload_date': '20151119',
1556                 'uploader_id': 'IronSoulElf',
1557                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1558                 'uploader': 'IronSoulElf',
1559                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1560                 'track': 'Dark Walk',
1561                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1562                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1563                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1564                 'categories': ['Film & Animation'],
1565                 'view_count': int,
1566                 'live_status': 'not_live',
1567                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1568                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1569                 'tags': 'count:13',
1570                 'availability': 'public',
1571                 'channel': 'IronSoulElf',
1572                 'playable_in_embed': True,
1573                 'like_count': int,
1574                 'age_limit': 0,
1575                 'channel_follower_count': int
1576             },
1577             'params': {
1578                 'skip_download': True,
1579             },
1580         },
1581         {
1582             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1583             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1584             'only_matching': True,
1585         },
1586         {
1587             # Video with yt:stretch=17:0
1588             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1589             'info_dict': {
1590                 'id': 'Q39EVAstoRM',
1591                 'ext': 'mp4',
1592                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1593                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1594                 'upload_date': '20151107',
1595                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1596                 'uploader': 'CH GAMER DROID',
1597             },
1598             'params': {
1599                 'skip_download': True,
1600             },
1601             'skip': 'This video does not exist.',
1602         },
1603         {
1604             # Video with incomplete 'yt:stretch=16:'
1605             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1606             'only_matching': True,
1607         },
1608         {
1609             # Video licensed under Creative Commons
1610             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1611             'info_dict': {
1612                 'id': 'M4gD1WSo5mA',
1613                 'ext': 'mp4',
1614                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1615                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1616                 'duration': 721,
1617                 'upload_date': '20150128',
1618                 'uploader_id': 'BerkmanCenter',
1619                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1620                 'uploader': 'The Berkman Klein Center for Internet & Society',
1621                 'license': 'Creative Commons Attribution license (reuse allowed)',
1622                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1623                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1624                 'like_count': int,
1625                 'age_limit': 0,
1626                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1627                 'channel': 'The Berkman Klein Center for Internet & Society',
1628                 'availability': 'public',
1629                 'view_count': int,
1630                 'categories': ['Education'],
1631                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1632                 'live_status': 'not_live',
1633                 'playable_in_embed': True,
1634                 'channel_follower_count': int
1635             },
1636             'params': {
1637                 'skip_download': True,
1638             },
1639         },
1640         {
1641             # Channel-like uploader_url
1642             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1643             'info_dict': {
1644                 'id': 'eQcmzGIKrzg',
1645                 'ext': 'mp4',
1646                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1647                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1648                 'duration': 4060,
1649                 'upload_date': '20151120',
1650                 'uploader': 'Bernie Sanders',
1651                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1652                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1653                 'license': 'Creative Commons Attribution license (reuse allowed)',
1654                 'playable_in_embed': True,
1655                 'tags': 'count:12',
1656                 'like_count': int,
1657                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1658                 'age_limit': 0,
1659                 'availability': 'public',
1660                 'categories': ['News & Politics'],
1661                 'channel': 'Bernie Sanders',
1662                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1663                 'view_count': int,
1664                 'live_status': 'not_live',
1665                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1666                 'channel_follower_count': int
1667             },
1668             'params': {
1669                 'skip_download': True,
1670             },
1671         },
1672         {
1673             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1674             'only_matching': True,
1675         },
1676         {
1677             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1678             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1679             'only_matching': True,
1680         },
1681         {
1682             # Rental video preview
1683             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1684             'info_dict': {
1685                 'id': 'uGpuVWrhIzE',
1686                 'ext': 'mp4',
1687                 'title': 'Piku - Trailer',
1688                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1689                 'upload_date': '20150811',
1690                 'uploader': 'FlixMatrix',
1691                 'uploader_id': 'FlixMatrixKaravan',
1692                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1693                 'license': 'Standard YouTube License',
1694             },
1695             'params': {
1696                 'skip_download': True,
1697             },
1698             'skip': 'This video is not available.',
1699         },
1700         {
1701             # YouTube Red video with episode data
1702             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1703             'info_dict': {
1704                 'id': 'iqKdEhx-dD4',
1705                 'ext': 'mp4',
1706                 'title': 'Isolation - Mind Field (Ep 1)',
1707                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1708                 'duration': 2085,
1709                 'upload_date': '20170118',
1710                 'uploader': 'Vsauce',
1711                 'uploader_id': 'Vsauce',
1712                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1713                 'series': 'Mind Field',
1714                 'season_number': 1,
1715                 'episode_number': 1,
1716                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1717                 'tags': 'count:12',
1718                 'view_count': int,
1719                 'availability': 'public',
1720                 'age_limit': 0,
1721                 'channel': 'Vsauce',
1722                 'episode': 'Episode 1',
1723                 'categories': ['Entertainment'],
1724                 'season': 'Season 1',
1725                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1726                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1727                 'like_count': int,
1728                 'playable_in_embed': True,
1729                 'live_status': 'not_live',
1730                 'channel_follower_count': int
1731             },
1732             'params': {
1733                 'skip_download': True,
1734             },
1735             'expected_warnings': [
1736                 'Skipping DASH manifest',
1737             ],
1738         },
1739         {
1740             # The following content has been identified by the YouTube community
1741             # as inappropriate or offensive to some audiences.
1742             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1743             'info_dict': {
1744                 'id': '6SJNVb0GnPI',
1745                 'ext': 'mp4',
1746                 'title': 'Race Differences in Intelligence',
1747                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1748                 'duration': 965,
1749                 'upload_date': '20140124',
1750                 'uploader': 'New Century Foundation',
1751                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1752                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1753             },
1754             'params': {
1755                 'skip_download': True,
1756             },
1757             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1758         },
1759         {
1760             # itag 212
1761             'url': '1t24XAntNCY',
1762             'only_matching': True,
1763         },
1764         {
1765             # geo restricted to JP
1766             'url': 'sJL6WA-aGkQ',
1767             'only_matching': True,
1768         },
1769         {
1770             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1771             'only_matching': True,
1772         },
1773         {
1774             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1775             'only_matching': True,
1776         },
1777         {
1778             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1779             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1780             'only_matching': True,
1781         },
1782         {
1783             # DRM protected
1784             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1785             'only_matching': True,
1786         },
1787         {
1788             # Video with unsupported adaptive stream type formats
1789             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1790             'info_dict': {
1791                 'id': 'Z4Vy8R84T1U',
1792                 'ext': 'mp4',
1793                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1794                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1795                 'duration': 433,
1796                 'upload_date': '20130923',
1797                 'uploader': 'Amelia Putri Harwita',
1798                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1799                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1800                 'formats': 'maxcount:10',
1801             },
1802             'params': {
1803                 'skip_download': True,
1804                 'youtube_include_dash_manifest': False,
1805             },
1806             'skip': 'not actual anymore',
1807         },
1808         {
1809             # Youtube Music Auto-generated description
1810             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1811             'info_dict': {
1812                 'id': 'MgNrAu2pzNs',
1813                 'ext': 'mp4',
1814                 'title': 'Voyeur Girl',
1815                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1816                 'upload_date': '20190312',
1817                 'uploader': 'Stephen - Topic',
1818                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1819                 'artist': 'Stephen',
1820                 'track': 'Voyeur Girl',
1821                 'album': 'it\'s too much love to know my dear',
1822                 'release_date': '20190313',
1823                 'release_year': 2019,
1824                 'alt_title': 'Voyeur Girl',
1825                 'view_count': int,
1826                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1827                 'playable_in_embed': True,
1828                 'like_count': int,
1829                 'categories': ['Music'],
1830                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1831                 'channel': 'Stephen',
1832                 'availability': 'public',
1833                 'creator': 'Stephen',
1834                 'duration': 169,
1835                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1836                 'age_limit': 0,
1837                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1838                 'tags': 'count:11',
1839                 'live_status': 'not_live',
1840                 'channel_follower_count': int
1841             },
1842             'params': {
1843                 'skip_download': True,
1844             },
1845         },
1846         {
1847             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1848             'only_matching': True,
1849         },
1850         {
1851             # invalid -> valid video id redirection
1852             'url': 'DJztXj2GPfl',
1853             'info_dict': {
1854                 'id': 'DJztXj2GPfk',
1855                 'ext': 'mp4',
1856                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1857                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1858                 'upload_date': '20090125',
1859                 'uploader': 'Prochorowka',
1860                 'uploader_id': 'Prochorowka',
1861                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1862                 'artist': 'Panjabi MC',
1863                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1864                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1865             },
1866             'params': {
1867                 'skip_download': True,
1868             },
1869             'skip': 'Video unavailable',
1870         },
1871         {
1872             # empty description results in an empty string
1873             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1874             'info_dict': {
1875                 'id': 'x41yOUIvK2k',
1876                 'ext': 'mp4',
1877                 'title': 'IMG 3456',
1878                 'description': '',
1879                 'upload_date': '20170613',
1880                 'uploader_id': 'ElevageOrVert',
1881                 'uploader': 'ElevageOrVert',
1882                 'view_count': int,
1883                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1884                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1885                 'like_count': int,
1886                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1887                 'tags': [],
1888                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1889                 'availability': 'public',
1890                 'age_limit': 0,
1891                 'categories': ['Pets & Animals'],
1892                 'duration': 7,
1893                 'playable_in_embed': True,
1894                 'live_status': 'not_live',
1895                 'channel': 'ElevageOrVert',
1896                 'channel_follower_count': int
1897             },
1898             'params': {
1899                 'skip_download': True,
1900             },
1901         },
1902         {
1903             # with '};' inside yt initial data (see [1])
1904             # see [2] for an example with '};' inside ytInitialPlayerResponse
1905             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1906             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1907             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1908             'info_dict': {
1909                 'id': 'CHqg6qOn4no',
1910                 'ext': 'mp4',
1911                 'title': 'Part 77   Sort a list of simple types in c#',
1912                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1913                 'upload_date': '20130831',
1914                 'uploader_id': 'kudvenkat',
1915                 'uploader': 'kudvenkat',
1916                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1917                 'like_count': int,
1918                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1919                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1920                 'live_status': 'not_live',
1921                 'categories': ['Education'],
1922                 'availability': 'public',
1923                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1924                 'tags': 'count:12',
1925                 'playable_in_embed': True,
1926                 'age_limit': 0,
1927                 'view_count': int,
1928                 'duration': 522,
1929                 'channel': 'kudvenkat',
1930                 'channel_follower_count': int
1931             },
1932             'params': {
1933                 'skip_download': True,
1934             },
1935         },
1936         {
1937             # another example of '};' in ytInitialData
1938             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1939             'only_matching': True,
1940         },
1941         {
1942             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1943             'only_matching': True,
1944         },
1945         {
1946             # https://github.com/ytdl-org/youtube-dl/pull/28094
1947             'url': 'OtqTfy26tG0',
1948             'info_dict': {
1949                 'id': 'OtqTfy26tG0',
1950                 'ext': 'mp4',
1951                 'title': 'Burn Out',
1952                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1953                 'upload_date': '20141120',
1954                 'uploader': 'The Cinematic Orchestra - Topic',
1955                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1956                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1957                 'artist': 'The Cinematic Orchestra',
1958                 'track': 'Burn Out',
1959                 'album': 'Every Day',
1960                 'like_count': int,
1961                 'live_status': 'not_live',
1962                 'alt_title': 'Burn Out',
1963                 'duration': 614,
1964                 'age_limit': 0,
1965                 'view_count': int,
1966                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1967                 'creator': 'The Cinematic Orchestra',
1968                 'channel': 'The Cinematic Orchestra',
1969                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1970                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1971                 'availability': 'public',
1972                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1973                 'categories': ['Music'],
1974                 'playable_in_embed': True,
1975                 'channel_follower_count': int
1976             },
1977             'params': {
1978                 'skip_download': True,
1979             },
1980         },
1981         {
1982             # controversial video, only works with bpctr when authenticated with cookies
1983             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1984             'only_matching': True,
1985         },
1986         {
1987             # controversial video, requires bpctr/contentCheckOk
1988             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1989             'info_dict': {
1990                 'id': 'SZJvDhaSDnc',
1991                 'ext': 'mp4',
1992                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1993                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1994                 'uploader': 'CBS Mornings',
1995                 'uploader_id': 'CBSThisMorning',
1996                 'upload_date': '20140716',
1997                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1998                 'duration': 170,
1999                 'categories': ['News & Politics'],
2000                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2001                 'view_count': int,
2002                 'channel': 'CBS Mornings',
2003                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2004                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2005                 'age_limit': 18,
2006                 'availability': 'needs_auth',
2007                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2008                 'like_count': int,
2009                 'live_status': 'not_live',
2010                 'playable_in_embed': True,
2011                 'channel_follower_count': int
2012             }
2013         },
2014         {
2015             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2016             'url': 'cBvYw8_A0vQ',
2017             'info_dict': {
2018                 'id': 'cBvYw8_A0vQ',
2019                 'ext': 'mp4',
2020                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2021                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2022                 'upload_date': '20201120',
2023                 'uploader': 'Walk around Japan',
2024                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2025                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2026                 'duration': 1456,
2027                 'categories': ['Travel & Events'],
2028                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2029                 'view_count': int,
2030                 'channel': 'Walk around Japan',
2031                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2032                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2033                 'age_limit': 0,
2034                 'availability': 'public',
2035                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2036                 'live_status': 'not_live',
2037                 'playable_in_embed': True,
2038                 'channel_follower_count': int
2039             },
2040             'params': {
2041                 'skip_download': True,
2042             },
2043         }, {
2044             # Has multiple audio streams
2045             'url': 'WaOKSUlf4TM',
2046             'only_matching': True
2047         }, {
2048             # Requires Premium: has format 141 when requested using YTM url
2049             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2050             'only_matching': True
2051         }, {
2052             # multiple subtitles with same lang_code
2053             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2054             'only_matching': True,
2055         }, {
2056             # Force use android client fallback
2057             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2058             'info_dict': {
2059                 'id': 'YOelRv7fMxY',
2060                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2061                 'ext': '3gp',
2062                 'upload_date': '20210624',
2063                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2064                 'uploader': 'colinfurze',
2065                 'uploader_id': 'colinfurze',
2066                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2067                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2068                 'duration': 596,
2069                 'categories': ['Entertainment'],
2070                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2071                 'view_count': int,
2072                 'channel': 'colinfurze',
2073                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2074                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2075                 'age_limit': 0,
2076                 'availability': 'public',
2077                 'like_count': int,
2078                 'live_status': 'not_live',
2079                 'playable_in_embed': True,
2080                 'channel_follower_count': int
2081             },
2082             'params': {
2083                 'format': '17',  # 3gp format available on android
2084                 'extractor_args': {'youtube': {'player_client': ['android']}},
2085             },
2086         },
2087         {
2088             # Skip download of additional client configs (remix client config in this case)
2089             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2090             'only_matching': True,
2091             'params': {
2092                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2093             },
2094         }, {
2095             # shorts
2096             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2097             'only_matching': True,
2098         }, {
2099             'note': 'Storyboards',
2100             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2101             'info_dict': {
2102                 'id': '5KLPxDtMqe8',
2103                 'ext': 'mhtml',
2104                 'format_id': 'sb0',
2105                 'title': 'Your Brain is Plastic',
2106                 'uploader_id': 'scishow',
2107                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2108                 'upload_date': '20140324',
2109                 'uploader': 'SciShow',
2110                 'like_count': int,
2111                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2112                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2113                 'view_count': int,
2114                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2115                 'playable_in_embed': True,
2116                 'tags': 'count:12',
2117                 'uploader_url': 'http://www.youtube.com/user/scishow',
2118                 'availability': 'public',
2119                 'channel': 'SciShow',
2120                 'live_status': 'not_live',
2121                 'duration': 248,
2122                 'categories': ['Education'],
2123                 'age_limit': 0,
2124                 'channel_follower_count': int
2125             }, 'params': {'format': 'mhtml', 'skip_download': True}
2126         }, {
2127             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2128             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2129             'info_dict': {
2130                 'id': '2NUZ8W2llS4',
2131                 'ext': 'mp4',
2132                 'title': 'The NP that test your phone performance 🙂',
2133                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2134                 'uploader': 'Leon Nguyen',
2135                 'uploader_id': 'VNSXIII',
2136                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2137                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2138                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2139                 'duration': 21,
2140                 'view_count': int,
2141                 'age_limit': 0,
2142                 'categories': ['Gaming'],
2143                 'tags': 'count:23',
2144                 'playable_in_embed': True,
2145                 'live_status': 'not_live',
2146                 'upload_date': '20220103',
2147                 'like_count': int,
2148                 'availability': 'public',
2149                 'channel': 'Leon Nguyen',
2150                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2151                 'channel_follower_count': int
2152             }
2153         }, {
2154             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2155             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2156             'info_dict': {
2157                 'id': 'mzZzzBU6lrM',
2158                 'ext': 'mp4',
2159                 'title': 'I Met GeorgeNotFound In Real Life...',
2160                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2161                 'uploader': 'Quackity',
2162                 'uploader_id': 'QuackityHQ',
2163                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2164                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2165                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2166                 'duration': 955,
2167                 'view_count': int,
2168                 'age_limit': 0,
2169                 'categories': ['Entertainment'],
2170                 'tags': 'count:26',
2171                 'playable_in_embed': True,
2172                 'live_status': 'not_live',
2173                 'release_timestamp': 1641172509,
2174                 'release_date': '20220103',
2175                 'upload_date': '20220103',
2176                 'like_count': int,
2177                 'availability': 'public',
2178                 'channel': 'Quackity',
2179                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2180                 'channel_follower_count': int
2181             }
2182         },
2183         {   # continuous livestream. Microformat upload date should be preferred.
2184             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2185             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2186             'info_dict': {
2187                 'id': 'kgx4WGK0oNU',
2188                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2189                 'ext': 'mp4',
2190                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2191                 'availability': 'public',
2192                 'age_limit': 0,
2193                 'release_timestamp': 1637975704,
2194                 'upload_date': '20210619',
2195                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2196                 'live_status': 'is_live',
2197                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2198                 'uploader': '阿鲍Abao',
2199                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2200                 'channel': 'Abao in Tokyo',
2201                 'channel_follower_count': int,
2202                 'release_date': '20211127',
2203                 'tags': 'count:39',
2204                 'categories': ['People & Blogs'],
2205                 'like_count': int,
2206                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2207                 'view_count': int,
2208                 'playable_in_embed': True,
2209                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2210             },
2211             'params': {'skip_download': True}
2212         }, {
2213             # Story. Requires specific player params to work.
2214             # Note: stories get removed after some period of time
2215             'url': 'https://www.youtube.com/watch?v=yN3x1t3sieA',
2216             'info_dict': {
2217                 'id': 'yN3x1t3sieA',
2218                 'ext': 'mp4',
2219                 'uploader': 'Linus Tech Tips',
2220                 'duration': 13,
2221                 'channel': 'Linus Tech Tips',
2222                 'playable_in_embed': True,
2223                 'tags': [],
2224                 'age_limit': 0,
2225                 'uploader_url': 'http://www.youtube.com/user/LinusTechTips',
2226                 'upload_date': '20220402',
2227                 'thumbnail': 'https://i.ytimg.com/vi_webp/yN3x1t3sieA/maxresdefault.webp',
2228                 'title': 'Story',
2229                 'live_status': 'not_live',
2230                 'uploader_id': 'LinusTechTips',
2231                 'view_count': int,
2232                 'description': '',
2233                 'channel_id': 'UCXuqSBlHAE6Xw-yeJA0Tunw',
2234                 'categories': ['Science & Technology'],
2235                 'channel_url': 'https://www.youtube.com/channel/UCXuqSBlHAE6Xw-yeJA0Tunw',
2236                 'availability': 'unlisted',
2237             }
2238         }
2239     ]
2240
2241     @classmethod
2242     def suitable(cls, url):
2243         from ..utils import parse_qs
2244
2245         qs = parse_qs(url)
2246         if qs.get('list', [None])[0]:
2247             return False
2248         return super().suitable(url)
2249
2250     def __init__(self, *args, **kwargs):
2251         super().__init__(*args, **kwargs)
2252         self._code_cache = {}
2253         self._player_cache = {}
2254
2255     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2256         lock = threading.Lock()
2257
2258         is_live = True
2259         start_time = time.time()
2260         formats = [f for f in formats if f.get('is_from_start')]
2261
2262         def refetch_manifest(format_id, delay):
2263             nonlocal formats, start_time, is_live
2264             if time.time() <= start_time + delay:
2265                 return
2266
2267             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2268             video_details = traverse_obj(
2269                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2270             microformats = traverse_obj(
2271                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2272                 expected_type=dict, default=[])
2273             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2274             start_time = time.time()
2275
2276         def mpd_feed(format_id, delay):
2277             """
2278             @returns (manifest_url, manifest_stream_number, is_live) or None
2279             """
2280             with lock:
2281                 refetch_manifest(format_id, delay)
2282
2283             f = next((f for f in formats if f['format_id'] == format_id), None)
2284             if not f:
2285                 if not is_live:
2286                     self.to_screen(f'{video_id}: Video is no longer live')
2287                 else:
2288                     self.report_warning(
2289                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2290                 return None
2291             return f['manifest_url'], f['manifest_stream_number'], is_live
2292
2293         for f in formats:
2294             f['is_live'] = True
2295             f['protocol'] = 'http_dash_segments_generator'
2296             f['fragments'] = functools.partial(
2297                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2298
2299     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2300         FETCH_SPAN, MAX_DURATION = 5, 432000
2301
2302         mpd_url, stream_number, is_live = None, None, True
2303
2304         begin_index = 0
2305         download_start_time = ctx.get('start') or time.time()
2306
2307         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2308         if lack_early_segments:
2309             self.report_warning(bug_reports_message(
2310                 'Starting download from the last 120 hours of the live stream since '
2311                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2312             lack_early_segments = True
2313
2314         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2315         fragments, fragment_base_url = None, None
2316
2317         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2318             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2319             # Obtain from MPD's maximum seq value
2320             old_mpd_url = mpd_url
2321             last_error = ctx.pop('last_error', None)
2322             expire_fast = immediate or last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2323             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2324                                                or (mpd_url, stream_number, False))
2325             if not refresh_sequence:
2326                 if expire_fast and not is_live:
2327                     return False, last_seq
2328                 elif old_mpd_url == mpd_url:
2329                     return True, last_seq
2330             try:
2331                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2332                     mpd_url, None, note=False, errnote=False, fatal=False)
2333             except ExtractorError:
2334                 fmts = None
2335             if not fmts:
2336                 no_fragment_score += 2
2337                 return False, last_seq
2338             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2339             fragments = fmt_info['fragments']
2340             fragment_base_url = fmt_info['fragment_base_url']
2341             assert fragment_base_url
2342
2343             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2344             return True, _last_seq
2345
2346         while is_live:
2347             fetch_time = time.time()
2348             if no_fragment_score > 30:
2349                 return
2350             if last_segment_url:
2351                 # Obtain from "X-Head-Seqnum" header value from each segment
2352                 try:
2353                     urlh = self._request_webpage(
2354                         last_segment_url, None, note=False, errnote=False, fatal=False)
2355                 except ExtractorError:
2356                     urlh = None
2357                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2358                 if last_seq is None:
2359                     no_fragment_score += 2
2360                     last_segment_url = None
2361                     continue
2362             else:
2363                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2364                 no_fragment_score += 2
2365                 if not should_continue:
2366                     continue
2367
2368             if known_idx > last_seq:
2369                 last_segment_url = None
2370                 continue
2371
2372             last_seq += 1
2373
2374             if begin_index < 0 and known_idx < 0:
2375                 # skip from the start when it's negative value
2376                 known_idx = last_seq + begin_index
2377             if lack_early_segments:
2378                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2379             try:
2380                 for idx in range(known_idx, last_seq):
2381                     # do not update sequence here or you'll get skipped some part of it
2382                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2383                     if not should_continue:
2384                         known_idx = idx - 1
2385                         raise ExtractorError('breaking out of outer loop')
2386                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2387                     yield {
2388                         'url': last_segment_url,
2389                     }
2390                 if known_idx == last_seq:
2391                     no_fragment_score += 5
2392                 else:
2393                     no_fragment_score = 0
2394                 known_idx = last_seq
2395             except ExtractorError:
2396                 continue
2397
2398             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2399
2400     def _extract_player_url(self, *ytcfgs, webpage=None):
2401         player_url = traverse_obj(
2402             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2403             get_all=False, expected_type=compat_str)
2404         if not player_url:
2405             return
2406         return urljoin('https://www.youtube.com', player_url)
2407
2408     def _download_player_url(self, video_id, fatal=False):
2409         res = self._download_webpage(
2410             'https://www.youtube.com/iframe_api',
2411             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2412         if res:
2413             player_version = self._search_regex(
2414                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2415             if player_version:
2416                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2417
2418     def _signature_cache_id(self, example_sig):
2419         """ Return a string representation of a signature """
2420         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2421
2422     @classmethod
2423     def _extract_player_info(cls, player_url):
2424         for player_re in cls._PLAYER_INFO_RE:
2425             id_m = re.search(player_re, player_url)
2426             if id_m:
2427                 break
2428         else:
2429             raise ExtractorError('Cannot identify player %r' % player_url)
2430         return id_m.group('id')
2431
2432     def _load_player(self, video_id, player_url, fatal=True):
2433         player_id = self._extract_player_info(player_url)
2434         if player_id not in self._code_cache:
2435             code = self._download_webpage(
2436                 player_url, video_id, fatal=fatal,
2437                 note='Downloading player ' + player_id,
2438                 errnote='Download of %s failed' % player_url)
2439             if code:
2440                 self._code_cache[player_id] = code
2441         return self._code_cache.get(player_id)
2442
2443     def _extract_signature_function(self, video_id, player_url, example_sig):
2444         player_id = self._extract_player_info(player_url)
2445
2446         # Read from filesystem cache
2447         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2448         assert os.path.basename(func_id) == func_id
2449
2450         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
2451         if cache_spec is not None:
2452             return lambda s: ''.join(s[i] for i in cache_spec)
2453
2454         code = self._load_player(video_id, player_url)
2455         if code:
2456             res = self._parse_sig_js(code)
2457
2458             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2459             cache_res = res(test_string)
2460             cache_spec = [ord(c) for c in cache_res]
2461
2462             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
2463             return res
2464
2465     def _print_sig_code(self, func, example_sig):
2466         if not self.get_param('youtube_print_sig_code'):
2467             return
2468
2469         def gen_sig_code(idxs):
2470             def _genslice(start, end, step):
2471                 starts = '' if start == 0 else str(start)
2472                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2473                 steps = '' if step == 1 else (':%d' % step)
2474                 return f's[{starts}{ends}{steps}]'
2475
2476             step = None
2477             # Quelch pyflakes warnings - start will be set when step is set
2478             start = '(Never used)'
2479             for i, prev in zip(idxs[1:], idxs[:-1]):
2480                 if step is not None:
2481                     if i - prev == step:
2482                         continue
2483                     yield _genslice(start, prev, step)
2484                     step = None
2485                     continue
2486                 if i - prev in [-1, 1]:
2487                     step = i - prev
2488                     start = prev
2489                     continue
2490                 else:
2491                     yield 's[%d]' % prev
2492             if step is None:
2493                 yield 's[%d]' % i
2494             else:
2495                 yield _genslice(start, i, step)
2496
2497         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2498         cache_res = func(test_string)
2499         cache_spec = [ord(c) for c in cache_res]
2500         expr_code = ' + '.join(gen_sig_code(cache_spec))
2501         signature_id_tuple = '(%s)' % (
2502             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2503         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2504                 '    return %s\n') % (signature_id_tuple, expr_code)
2505         self.to_screen('Extracted signature function:\n' + code)
2506
2507     def _parse_sig_js(self, jscode):
2508         funcname = self._search_regex(
2509             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2510              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2511              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2512              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2513              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2514              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2515              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2516              # Obsolete patterns
2517              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2518              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2519              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2520              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2521              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2522              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2523              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2524              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2525             jscode, 'Initial JS player signature function name', group='sig')
2526
2527         jsi = JSInterpreter(jscode)
2528         initial_function = jsi.extract_function(funcname)
2529         return lambda s: initial_function([s])
2530
2531     def _decrypt_signature(self, s, video_id, player_url):
2532         """Turn the encrypted s field into a working signature"""
2533         try:
2534             player_id = (player_url, self._signature_cache_id(s))
2535             if player_id not in self._player_cache:
2536                 func = self._extract_signature_function(video_id, player_url, s)
2537                 self._player_cache[player_id] = func
2538             func = self._player_cache[player_id]
2539             self._print_sig_code(func, s)
2540             return func(s)
2541         except Exception as e:
2542             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2543
2544     def _decrypt_nsig(self, s, video_id, player_url):
2545         """Turn the encrypted n field into a working signature"""
2546         if player_url is None:
2547             raise ExtractorError('Cannot decrypt nsig without player_url')
2548         player_url = urljoin('https://www.youtube.com', player_url)
2549
2550         sig_id = ('nsig_value', s)
2551         if sig_id in self._player_cache:
2552             return self._player_cache[sig_id]
2553
2554         try:
2555             player_id = ('nsig', player_url)
2556             if player_id not in self._player_cache:
2557                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2558             func = self._player_cache[player_id]
2559             self._player_cache[sig_id] = func(s)
2560             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2561             return self._player_cache[sig_id]
2562         except Exception as e:
2563             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2564
2565     def _extract_n_function_name(self, jscode):
2566         nfunc, idx = self._search_regex(
2567             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2568             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2569         if not idx:
2570             return nfunc
2571         return json.loads(js_to_json(self._search_regex(
2572             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2573             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2574
2575     def _extract_n_function(self, video_id, player_url):
2576         player_id = self._extract_player_info(player_url)
2577         func_code = self._downloader.cache.load('youtube-nsig', player_id)
2578
2579         if func_code:
2580             jsi = JSInterpreter(func_code)
2581         else:
2582             jscode = self._load_player(video_id, player_url)
2583             funcname = self._extract_n_function_name(jscode)
2584             jsi = JSInterpreter(jscode)
2585             func_code = jsi.extract_function_code(funcname)
2586             self._downloader.cache.store('youtube-nsig', player_id, func_code)
2587
2588         if self.get_param('youtube_print_sig_code'):
2589             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2590
2591         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2592
2593     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2594         """
2595         Extract signatureTimestamp (sts)
2596         Required to tell API what sig/player version is in use.
2597         """
2598         sts = None
2599         if isinstance(ytcfg, dict):
2600             sts = int_or_none(ytcfg.get('STS'))
2601
2602         if not sts:
2603             # Attempt to extract from player
2604             if player_url is None:
2605                 error_msg = 'Cannot extract signature timestamp without player_url.'
2606                 if fatal:
2607                     raise ExtractorError(error_msg)
2608                 self.report_warning(error_msg)
2609                 return
2610             code = self._load_player(video_id, player_url, fatal=fatal)
2611             if code:
2612                 sts = int_or_none(self._search_regex(
2613                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2614                     'JS player signature timestamp', group='sts', fatal=fatal))
2615         return sts
2616
2617     def _mark_watched(self, video_id, player_responses):
2618         playback_url = get_first(
2619             player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2620             expected_type=url_or_none)
2621         if not playback_url:
2622             self.report_warning('Unable to mark watched')
2623             return
2624         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2625         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2626
2627         # cpn generation algorithm is reverse engineered from base.js.
2628         # In fact it works even with dummy cpn.
2629         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2630         cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2631
2632         qs.update({
2633             'ver': ['2'],
2634             'cpn': [cpn],
2635         })
2636         playback_url = compat_urlparse.urlunparse(
2637             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2638
2639         self._download_webpage(
2640             playback_url, video_id, 'Marking watched',
2641             'Unable to mark watched', fatal=False)
2642
2643     @staticmethod
2644     def _extract_urls(webpage):
2645         # Embedded YouTube player
2646         entries = [
2647             unescapeHTML(mobj.group('url'))
2648             for mobj in re.finditer(r'''(?x)
2649             (?:
2650                 <iframe[^>]+?src=|
2651                 data-video-url=|
2652                 <embed[^>]+?src=|
2653                 embedSWF\(?:\s*|
2654                 <object[^>]+data=|
2655                 new\s+SWFObject\(
2656             )
2657             (["\'])
2658                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2659                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2660             \1''', webpage)]
2661
2662         # lazyYT YouTube embed
2663         entries.extend(list(map(
2664             unescapeHTML,
2665             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2666
2667         # Wordpress "YouTube Video Importer" plugin
2668         matches = re.findall(r'''(?x)<div[^>]+
2669             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2670             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2671         entries.extend(m[-1] for m in matches)
2672
2673         return entries
2674
2675     @staticmethod
2676     def _extract_url(webpage):
2677         urls = YoutubeIE._extract_urls(webpage)
2678         return urls[0] if urls else None
2679
2680     @classmethod
2681     def extract_id(cls, url):
2682         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2683         if mobj is None:
2684             raise ExtractorError('Invalid URL: %s' % url)
2685         return mobj.group('id')
2686
2687     def _extract_chapters_from_json(self, data, duration):
2688         chapter_list = traverse_obj(
2689             data, (
2690                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2691                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2692             ), expected_type=list)
2693
2694         return self._extract_chapters(
2695             chapter_list,
2696             chapter_time=lambda chapter: float_or_none(
2697                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2698             chapter_title=lambda chapter: traverse_obj(
2699                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2700             duration=duration)
2701
2702     def _extract_chapters_from_engagement_panel(self, data, duration):
2703         content_list = traverse_obj(
2704             data,
2705             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2706             expected_type=list, default=[])
2707         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2708         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2709
2710         return next((
2711             filter(None, (
2712                 self._extract_chapters(
2713                     traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2714                     chapter_time, chapter_title, duration)
2715                 for contents in content_list
2716             ))), [])
2717
2718     @staticmethod
2719     def _extract_chapters_from_description(description, duration):
2720         chapters = [{'start_time': 0}]
2721         for timestamp, title in re.findall(
2722                 r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''):
2723             start = parse_duration(timestamp)
2724             if start and title and chapters[-1]['start_time'] < start < duration:
2725                 chapters[-1]['end_time'] = start
2726                 chapters.append({
2727                     'start_time': start,
2728                     'title': title,
2729                 })
2730         chapters[-1]['end_time'] = duration
2731         return chapters[1:]
2732
2733     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration):
2734         chapters = []
2735         last_chapter = {'start_time': 0}
2736         for idx, chapter in enumerate(chapter_list or []):
2737             title = chapter_title(chapter)
2738             start_time = chapter_time(chapter)
2739             if start_time is None:
2740                 continue
2741             last_chapter['end_time'] = start_time
2742             if start_time < last_chapter['start_time']:
2743                 if idx == 1:
2744                     chapters.pop()
2745                     self.report_warning('Invalid start time for chapter "%s"' % last_chapter['title'])
2746                 else:
2747                     self.report_warning(f'Invalid start time for chapter "{title}"')
2748                     continue
2749             last_chapter = {'start_time': start_time, 'title': title}
2750             chapters.append(last_chapter)
2751         last_chapter['end_time'] = duration
2752         return chapters
2753
2754     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
2755         return self._parse_json(self._search_regex(
2756             (fr'{regex}\s*{self._YT_INITIAL_BOUNDARY_RE}',
2757              regex), webpage, name, default='{}'), video_id, fatal=False)
2758
2759     def _extract_comment(self, comment_renderer, parent=None):
2760         comment_id = comment_renderer.get('commentId')
2761         if not comment_id:
2762             return
2763
2764         text = self._get_text(comment_renderer, 'contentText')
2765
2766         # note: timestamp is an estimate calculated from the current time and time_text
2767         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2768         author = self._get_text(comment_renderer, 'authorText')
2769         author_id = try_get(comment_renderer,
2770                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2771
2772         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2773                                                        lambda x: x['likeCount']), compat_str)) or 0
2774         author_thumbnail = try_get(comment_renderer,
2775                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2776
2777         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2778         is_favorited = 'creatorHeart' in (try_get(
2779             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2780         return {
2781             'id': comment_id,
2782             'text': text,
2783             'timestamp': timestamp,
2784             'time_text': time_text,
2785             'like_count': votes,
2786             'is_favorited': is_favorited,
2787             'author': author,
2788             'author_id': author_id,
2789             'author_thumbnail': author_thumbnail,
2790             'author_is_uploader': author_is_uploader,
2791             'parent': parent or 'root'
2792         }
2793
2794     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2795
2796         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2797
2798         def extract_header(contents):
2799             _continuation = None
2800             for content in contents:
2801                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2802                 expected_comment_count = self._get_count(
2803                     comments_header_renderer, 'countText', 'commentsCount')
2804
2805                 if expected_comment_count:
2806                     tracker['est_total'] = expected_comment_count
2807                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2808                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2809
2810                 sort_menu_item = try_get(
2811                     comments_header_renderer,
2812                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2813                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2814
2815                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2816                 if not _continuation:
2817                     continue
2818
2819                 sort_text = str_or_none(sort_menu_item.get('title'))
2820                 if not sort_text:
2821                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2822                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2823                 break
2824             return _continuation
2825
2826         def extract_thread(contents):
2827             if not parent:
2828                 tracker['current_page_thread'] = 0
2829             for content in contents:
2830                 if not parent and tracker['total_parent_comments'] >= max_parents:
2831                     yield
2832                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2833                 comment_renderer = get_first(
2834                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2835                     expected_type=dict, default={})
2836
2837                 comment = self._extract_comment(comment_renderer, parent)
2838                 if not comment:
2839                     continue
2840
2841                 tracker['running_total'] += 1
2842                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2843                 yield comment
2844
2845                 # Attempt to get the replies
2846                 comment_replies_renderer = try_get(
2847                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2848
2849                 if comment_replies_renderer:
2850                     tracker['current_page_thread'] += 1
2851                     comment_entries_iter = self._comment_entries(
2852                         comment_replies_renderer, ytcfg, video_id,
2853                         parent=comment.get('id'), tracker=tracker)
2854                     yield from itertools.islice(comment_entries_iter, min(
2855                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2856
2857         # Keeps track of counts across recursive calls
2858         if not tracker:
2859             tracker = dict(
2860                 running_total=0,
2861                 est_total=0,
2862                 current_page_thread=0,
2863                 total_parent_comments=0,
2864                 total_reply_comments=0)
2865
2866         # TODO: Deprecated
2867         # YouTube comments have a max depth of 2
2868         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2869         if max_depth:
2870             self._downloader.deprecation_warning(
2871                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2872         if max_depth == 1 and parent:
2873             return
2874
2875         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2876             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2877
2878         continuation = self._extract_continuation(root_continuation_data)
2879
2880         response = None
2881         is_forced_continuation = False
2882         is_first_continuation = parent is None
2883         if is_first_continuation and not continuation:
2884             # Sometimes you can get comments by generating the continuation yourself,
2885             # even if YouTube initially reports them being disabled - e.g. stories comments.
2886             # Note: if the comment section is actually disabled, YouTube may return a response with
2887             # required check_get_keys missing. So we will disable that check initially in this case.
2888             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2889             is_forced_continuation = True
2890
2891         for page_num in itertools.count(0):
2892             if not continuation:
2893                 break
2894             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2895             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2896             if page_num == 0:
2897                 if is_first_continuation:
2898                     note_prefix = 'Downloading comment section API JSON'
2899                 else:
2900                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2901                         tracker['current_page_thread'], comment_prog_str)
2902             else:
2903                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2904                     '       ' if parent else '', ' replies' if parent else '',
2905                     page_num, comment_prog_str)
2906
2907             response = self._extract_response(
2908                 item_id=None, query=continuation,
2909                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2910                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2911             is_forced_continuation = False
2912             continuation_contents = traverse_obj(
2913                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2914
2915             continuation = None
2916             for continuation_section in continuation_contents:
2917                 continuation_items = traverse_obj(
2918                     continuation_section,
2919                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2920                     get_all=False, expected_type=list) or []
2921                 if is_first_continuation:
2922                     continuation = extract_header(continuation_items)
2923                     is_first_continuation = False
2924                     if continuation:
2925                         break
2926                     continue
2927
2928                 for entry in extract_thread(continuation_items):
2929                     if not entry:
2930                         return
2931                     yield entry
2932                 continuation = self._extract_continuation({'contents': continuation_items})
2933                 if continuation:
2934                     break
2935
2936         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2937         if message and not parent and tracker['running_total'] == 0:
2938             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2939
2940     @staticmethod
2941     def _generate_comment_continuation(video_id):
2942         """
2943         Generates initial comment section continuation token from given video id
2944         """
2945         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2946         return base64.b64encode(token.encode()).decode()
2947
2948     def _get_comments(self, ytcfg, video_id, contents, webpage):
2949         """Entry for comment extraction"""
2950         def _real_comment_extract(contents):
2951             renderer = next((
2952                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2953                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2954             yield from self._comment_entries(renderer, ytcfg, video_id)
2955
2956         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2957         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2958
2959     @staticmethod
2960     def _get_checkok_params():
2961         return {'contentCheckOk': True, 'racyCheckOk': True}
2962
2963     @classmethod
2964     def _generate_player_context(cls, sts=None):
2965         context = {
2966             'html5Preference': 'HTML5_PREF_WANTS',
2967         }
2968         if sts is not None:
2969             context['signatureTimestamp'] = sts
2970         return {
2971             'playbackContext': {
2972                 'contentPlaybackContext': context
2973             },
2974             **cls._get_checkok_params()
2975         }
2976
2977     @staticmethod
2978     def _is_agegated(player_response):
2979         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2980             return True
2981
2982         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2983         AGE_GATE_REASONS = (
2984             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2985             'age_verification_required', 'age_check_required',  # status
2986         )
2987         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
2988
2989     @staticmethod
2990     def _is_unplayable(player_response):
2991         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
2992
2993     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
2994
2995         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
2996         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
2997         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
2998         headers = self.generate_api_headers(
2999             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3000
3001         yt_query = {
3002             'videoId': video_id,
3003             'params': '8AEB'  # enable stories
3004         }
3005         yt_query.update(self._generate_player_context(sts))
3006         return self._extract_response(
3007             item_id=video_id, ep='player', query=yt_query,
3008             ytcfg=player_ytcfg, headers=headers, fatal=True,
3009             default_client=client,
3010             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3011         ) or None
3012
3013     def _get_requested_clients(self, url, smuggled_data):
3014         requested_clients = []
3015         default = ['android', 'web']
3016         allowed_clients = sorted(
3017             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3018             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3019         for client in self._configuration_arg('player_client'):
3020             if client in allowed_clients:
3021                 requested_clients.append(client)
3022             elif client == 'default':
3023                 requested_clients.extend(default)
3024             elif client == 'all':
3025                 requested_clients.extend(allowed_clients)
3026             else:
3027                 self.report_warning(f'Skipping unsupported client {client}')
3028         if not requested_clients:
3029             requested_clients = default
3030
3031         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3032             requested_clients.extend(
3033                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3034
3035         return orderedSet(requested_clients)
3036
3037     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3038         initial_pr = None
3039         if webpage:
3040             initial_pr = self._extract_yt_initial_variable(
3041                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
3042                 video_id, 'initial player response')
3043
3044         all_clients = set(clients)
3045         clients = clients[::-1]
3046         prs = []
3047
3048         def append_client(*client_names):
3049             """ Append the first client name that exists but not already used """
3050             for client_name in client_names:
3051                 actual_client = _split_innertube_client(client_name)[0]
3052                 if actual_client in INNERTUBE_CLIENTS:
3053                     if actual_client not in all_clients:
3054                         clients.append(client_name)
3055                         all_clients.add(actual_client)
3056                         return
3057
3058         # Android player_response does not have microFormats which are needed for
3059         # extraction of some data. So we return the initial_pr with formats
3060         # stripped out even if not requested by the user
3061         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3062         if initial_pr:
3063             pr = dict(initial_pr)
3064             pr['streamingData'] = None
3065             prs.append(pr)
3066
3067         last_error = None
3068         tried_iframe_fallback = False
3069         player_url = None
3070         while clients:
3071             client, base_client, variant = _split_innertube_client(clients.pop())
3072             player_ytcfg = master_ytcfg if client == 'web' else {}
3073             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3074                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3075
3076             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3077             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3078             if 'js' in self._configuration_arg('player_skip'):
3079                 require_js_player = False
3080                 player_url = None
3081
3082             if not player_url and not tried_iframe_fallback and require_js_player:
3083                 player_url = self._download_player_url(video_id)
3084                 tried_iframe_fallback = True
3085
3086             try:
3087                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3088                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3089             except ExtractorError as e:
3090                 if last_error:
3091                     self.report_warning(last_error)
3092                 last_error = e
3093                 continue
3094
3095             if pr:
3096                 prs.append(pr)
3097
3098             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3099             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3100                 append_client(f'{base_client}_creator')
3101             elif self._is_agegated(pr):
3102                 if variant == 'tv_embedded':
3103                     append_client(f'{base_client}_embedded')
3104                 elif not variant:
3105                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3106
3107         if last_error:
3108             if not len(prs):
3109                 raise last_error
3110             self.report_warning(last_error)
3111         return prs, player_url
3112
3113     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3114         itags, stream_ids = {}, []
3115         itag_qualities, res_qualities = {}, {}
3116         q = qualities([
3117             # Normally tiny is the smallest video-only formats. But
3118             # audio-only formats with unknown quality may get tagged as tiny
3119             'tiny',
3120             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3121             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3122         ])
3123         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3124
3125         for fmt in streaming_formats:
3126             if fmt.get('targetDurationSec'):
3127                 continue
3128
3129             itag = str_or_none(fmt.get('itag'))
3130             audio_track = fmt.get('audioTrack') or {}
3131             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3132             if stream_id in stream_ids:
3133                 continue
3134
3135             quality = fmt.get('quality')
3136             height = int_or_none(fmt.get('height'))
3137             if quality == 'tiny' or not quality:
3138                 quality = fmt.get('audioQuality', '').lower() or quality
3139             # The 3gp format (17) in android client has a quality of "small",
3140             # but is actually worse than other formats
3141             if itag == '17':
3142                 quality = 'tiny'
3143             if quality:
3144                 if itag:
3145                     itag_qualities[itag] = quality
3146                 if height:
3147                     res_qualities[height] = quality
3148             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3149             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3150             # number of fragment that would subsequently requested with (`&sq=N`)
3151             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3152                 continue
3153
3154             fmt_url = fmt.get('url')
3155             if not fmt_url:
3156                 sc = compat_parse_qs(fmt.get('signatureCipher'))
3157                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3158                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3159                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3160                     continue
3161                 try:
3162                     fmt_url += '&%s=%s' % (
3163                         traverse_obj(sc, ('sp', -1)) or 'signature',
3164                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3165                     )
3166                 except ExtractorError as e:
3167                     self.report_warning('Signature extraction failed: Some formats may be missing', only_once=True)
3168                     self.write_debug(e, only_once=True)
3169                     continue
3170
3171             query = parse_qs(fmt_url)
3172             throttled = False
3173             if query.get('n'):
3174                 try:
3175                     fmt_url = update_url_query(fmt_url, {
3176                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3177                 except ExtractorError as e:
3178                     self.report_warning(
3179                         'nsig extraction failed: You may experience throttling for some formats\n'
3180                         f'n = {query["n"][0]} ; player = {player_url}', only_once=True)
3181                     self.write_debug(e, only_once=True)
3182                     throttled = True
3183
3184             if itag:
3185                 itags[itag] = 'https'
3186                 stream_ids.append(stream_id)
3187
3188             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3189             language_preference = (
3190                 10 if audio_track.get('audioIsDefault') and 10
3191                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3192                 else -1)
3193             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3194             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3195             # Make sure to avoid false positives with small duration differences.
3196             # Eg: __2ABJjxzNo, ySuUZEjARPY
3197             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3198             if is_damaged:
3199                 self.report_warning(
3200                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3201             dct = {
3202                 'asr': int_or_none(fmt.get('audioSampleRate')),
3203                 'filesize': int_or_none(fmt.get('contentLength')),
3204                 'format_id': itag,
3205                 'format_note': join_nonempty(
3206                     '%s%s' % (audio_track.get('displayName') or '',
3207                               ' (default)' if language_preference > 0 else ''),
3208                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3209                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3210                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3211                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3212                 'fps': int_or_none(fmt.get('fps')) or None,
3213                 'height': height,
3214                 'quality': q(quality),
3215                 'has_drm': bool(fmt.get('drmFamilies')),
3216                 'tbr': tbr,
3217                 'url': fmt_url,
3218                 'width': int_or_none(fmt.get('width')),
3219                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3220                                           'desc' if language_preference < -1 else ''),
3221                 'language_preference': language_preference,
3222                 # Strictly de-prioritize damaged and 3gp formats
3223                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3224             }
3225             mime_mobj = re.match(
3226                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3227             if mime_mobj:
3228                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3229                 dct.update(parse_codecs(mime_mobj.group(2)))
3230             no_audio = dct.get('acodec') == 'none'
3231             no_video = dct.get('vcodec') == 'none'
3232             if no_audio:
3233                 dct['vbr'] = tbr
3234             if no_video:
3235                 dct['abr'] = tbr
3236             if no_audio or no_video:
3237                 dct['downloader_options'] = {
3238                     # Youtube throttles chunks >~10M
3239                     'http_chunk_size': 10485760,
3240                 }
3241                 if dct.get('ext'):
3242                     dct['container'] = dct['ext'] + '_dash'
3243             yield dct
3244
3245         live_from_start = is_live and self.get_param('live_from_start')
3246         skip_manifests = self._configuration_arg('skip')
3247         if not self.get_param('youtube_include_hls_manifest', True):
3248             skip_manifests.append('hls')
3249         if not self.get_param('youtube_include_dash_manifest', True):
3250             skip_manifests.append('dash')
3251         get_dash = 'dash' not in skip_manifests and (
3252             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3253         get_hls = not live_from_start and 'hls' not in skip_manifests
3254
3255         def process_manifest_format(f, proto, itag):
3256             if itag in itags:
3257                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3258                     return False
3259                 itag = f'{itag}-{proto}'
3260             if itag:
3261                 f['format_id'] = itag
3262                 itags[itag] = proto
3263
3264             f['quality'] = next((
3265                 q(qdict[val])
3266                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3267                 if val in qdict), -1)
3268             return True
3269
3270         for sd in streaming_data:
3271             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3272             if hls_manifest_url:
3273                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3274                     if process_manifest_format(f, 'hls', self._search_regex(
3275                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3276                         yield f
3277
3278             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3279             if dash_manifest_url:
3280                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3281                     if process_manifest_format(f, 'dash', f['format_id']):
3282                         f['filesize'] = int_or_none(self._search_regex(
3283                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3284                         if live_from_start:
3285                             f['is_from_start'] = True
3286
3287                         yield f
3288
3289     def _extract_storyboard(self, player_responses, duration):
3290         spec = get_first(
3291             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3292         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3293         if not base_url:
3294             return
3295         L = len(spec) - 1
3296         for i, args in enumerate(spec):
3297             args = args.split('#')
3298             counts = list(map(int_or_none, args[:5]))
3299             if len(args) != 8 or not all(counts):
3300                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3301                 continue
3302             width, height, frame_count, cols, rows = counts
3303             N, sigh = args[6:]
3304
3305             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3306             fragment_count = frame_count / (cols * rows)
3307             fragment_duration = duration / fragment_count
3308             yield {
3309                 'format_id': f'sb{i}',
3310                 'format_note': 'storyboard',
3311                 'ext': 'mhtml',
3312                 'protocol': 'mhtml',
3313                 'acodec': 'none',
3314                 'vcodec': 'none',
3315                 'url': url,
3316                 'width': width,
3317                 'height': height,
3318                 'fragments': [{
3319                     'url': url.replace('$M', str(j)),
3320                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3321                 } for j in range(math.ceil(fragment_count))],
3322             }
3323
3324     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3325         webpage = None
3326         if 'webpage' not in self._configuration_arg('player_skip'):
3327             webpage = self._download_webpage(
3328                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3329
3330         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3331
3332         player_responses, player_url = self._extract_player_responses(
3333             self._get_requested_clients(url, smuggled_data),
3334             video_id, webpage, master_ytcfg)
3335
3336         return webpage, master_ytcfg, player_responses, player_url
3337
3338     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3339         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3340         is_live = get_first(video_details, 'isLive')
3341         if is_live is None:
3342             is_live = get_first(live_broadcast_details, 'isLiveNow')
3343
3344         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3345         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3346
3347         return live_broadcast_details, is_live, streaming_data, formats
3348
3349     def _real_extract(self, url):
3350         url, smuggled_data = unsmuggle_url(url, {})
3351         video_id = self._match_id(url)
3352
3353         base_url = self.http_scheme() + '//www.youtube.com/'
3354         webpage_url = base_url + 'watch?v=' + video_id
3355
3356         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3357
3358         playability_statuses = traverse_obj(
3359             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3360
3361         trailer_video_id = get_first(
3362             playability_statuses,
3363             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3364             expected_type=str)
3365         if trailer_video_id:
3366             return self.url_result(
3367                 trailer_video_id, self.ie_key(), trailer_video_id)
3368
3369         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3370                        if webpage else (lambda x: None))
3371
3372         video_details = traverse_obj(
3373             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3374         microformats = traverse_obj(
3375             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3376             expected_type=dict, default=[])
3377         video_title = (
3378             get_first(video_details, 'title')
3379             or self._get_text(microformats, (..., 'title'))
3380             or search_meta(['og:title', 'twitter:title', 'title']))
3381         video_description = get_first(video_details, 'shortDescription')
3382
3383         multifeed_metadata_list = get_first(
3384             player_responses,
3385             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3386             expected_type=str)
3387         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3388             if self.get_param('noplaylist'):
3389                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3390             else:
3391                 entries = []
3392                 feed_ids = []
3393                 for feed in multifeed_metadata_list.split(','):
3394                     # Unquote should take place before split on comma (,) since textual
3395                     # fields may contain comma as well (see
3396                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3397                     feed_data = compat_parse_qs(
3398                         compat_urllib_parse_unquote_plus(feed))
3399
3400                     def feed_entry(name):
3401                         return try_get(
3402                             feed_data, lambda x: x[name][0], compat_str)
3403
3404                     feed_id = feed_entry('id')
3405                     if not feed_id:
3406                         continue
3407                     feed_title = feed_entry('title')
3408                     title = video_title
3409                     if feed_title:
3410                         title += ' (%s)' % feed_title
3411                     entries.append({
3412                         '_type': 'url_transparent',
3413                         'ie_key': 'Youtube',
3414                         'url': smuggle_url(
3415                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3416                             {'force_singlefeed': True}),
3417                         'title': title,
3418                     })
3419                     feed_ids.append(feed_id)
3420                 self.to_screen(
3421                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3422                     % (', '.join(feed_ids), video_id))
3423                 return self.playlist_result(
3424                     entries, video_id, video_title, video_description)
3425
3426         duration = int_or_none(
3427             get_first(video_details, 'lengthSeconds')
3428             or get_first(microformats, 'lengthSeconds')
3429             or parse_duration(search_meta('duration'))) or None
3430
3431         if get_first(video_details, 'isPostLiveDvr'):
3432             self.write_debug('Video is in Post-Live Manifestless mode')
3433             if duration or 0 > 4 * 3600:
3434                 self.report_warning(
3435                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3436                     'This is a known issue and patches are welcome')
3437
3438         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3439             video_id, microformats, video_details, player_responses, player_url, duration)
3440
3441         if not formats:
3442             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3443                 self.report_drm(video_id)
3444             pemr = get_first(
3445                 playability_statuses,
3446                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3447             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3448             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3449             if subreason:
3450                 if subreason == 'The uploader has not made this video available in your country.':
3451                     countries = get_first(microformats, 'availableCountries')
3452                     if not countries:
3453                         regions_allowed = search_meta('regionsAllowed')
3454                         countries = regions_allowed.split(',') if regions_allowed else None
3455                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3456                 reason += f'. {subreason}'
3457             if reason:
3458                 self.raise_no_formats(reason, expected=True)
3459
3460         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3461         if not keywords and webpage:
3462             keywords = [
3463                 unescapeHTML(m.group('content'))
3464                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3465         for keyword in keywords:
3466             if keyword.startswith('yt:stretch='):
3467                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3468                 if mobj:
3469                     # NB: float is intentional for forcing float division
3470                     w, h = (float(v) for v in mobj.groups())
3471                     if w > 0 and h > 0:
3472                         ratio = w / h
3473                         for f in formats:
3474                             if f.get('vcodec') != 'none':
3475                                 f['stretched_ratio'] = ratio
3476                         break
3477         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3478         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3479         if thumbnail_url:
3480             thumbnails.append({
3481                 'url': thumbnail_url,
3482             })
3483         original_thumbnails = thumbnails.copy()
3484
3485         # The best resolution thumbnails sometimes does not appear in the webpage
3486         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3487         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3488         thumbnail_names = [
3489             # While the *1,*2,*3 thumbnails are just below their correspnding "*default" variants
3490             # in resolution, these are not the custom thumbnail. So de-prioritize them
3491             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3492             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3493         ]
3494         n_thumbnail_names = len(thumbnail_names)
3495         thumbnails.extend({
3496             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3497                 video_id=video_id, name=name, ext=ext,
3498                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3499         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3500         for thumb in thumbnails:
3501             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3502             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3503         self._remove_duplicate_formats(thumbnails)
3504         self._downloader._sort_thumbnails(original_thumbnails)
3505
3506         category = get_first(microformats, 'category') or search_meta('genre')
3507         channel_id = str_or_none(
3508             get_first(video_details, 'channelId')
3509             or get_first(microformats, 'externalChannelId')
3510             or search_meta('channelId'))
3511         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3512
3513         live_content = get_first(video_details, 'isLiveContent')
3514         is_upcoming = get_first(video_details, 'isUpcoming')
3515         if is_live is None:
3516             if is_upcoming or live_content is False:
3517                 is_live = False
3518         if is_upcoming is None and (live_content or is_live):
3519             is_upcoming = False
3520         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3521         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3522         if not duration and live_end_time and live_start_time:
3523             duration = live_end_time - live_start_time
3524
3525         if is_live and self.get_param('live_from_start'):
3526             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3527
3528         formats.extend(self._extract_storyboard(player_responses, duration))
3529
3530         # Source is given priority since formats that throttle are given lower source_preference
3531         # When throttling issue is fully fixed, remove this
3532         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3533
3534         info = {
3535             'id': video_id,
3536             'title': video_title,
3537             'formats': formats,
3538             'thumbnails': thumbnails,
3539             # The best thumbnail that we are sure exists. Prevents unnecessary
3540             # URL checking if user don't care about getting the best possible thumbnail
3541             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3542             'description': video_description,
3543             'uploader': get_first(video_details, 'author'),
3544             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3545             'uploader_url': owner_profile_url,
3546             'channel_id': channel_id,
3547             'channel_url': format_field(channel_id, template='https://www.youtube.com/channel/%s'),
3548             'duration': duration,
3549             'view_count': int_or_none(
3550                 get_first((video_details, microformats), (..., 'viewCount'))
3551                 or search_meta('interactionCount')),
3552             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3553             'age_limit': 18 if (
3554                 get_first(microformats, 'isFamilySafe') is False
3555                 or search_meta('isFamilyFriendly') == 'false'
3556                 or search_meta('og:restrictions:age') == '18+') else 0,
3557             'webpage_url': webpage_url,
3558             'categories': [category] if category else None,
3559             'tags': keywords,
3560             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3561             'is_live': is_live,
3562             'was_live': (False if is_live or is_upcoming or live_content is False
3563                          else None if is_live is None or is_upcoming is None
3564                          else live_content),
3565             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3566             'release_timestamp': live_start_time,
3567         }
3568
3569         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3570         if pctr:
3571             def get_lang_code(track):
3572                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3573                         or track.get('languageCode'))
3574
3575             # Converted into dicts to remove duplicates
3576             captions = {
3577                 get_lang_code(sub): sub
3578                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3579             translation_languages = {
3580                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3581                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3582
3583             def process_language(container, base_url, lang_code, sub_name, query):
3584                 lang_subs = container.setdefault(lang_code, [])
3585                 for fmt in self._SUBTITLE_FORMATS:
3586                     query.update({
3587                         'fmt': fmt,
3588                     })
3589                     lang_subs.append({
3590                         'ext': fmt,
3591                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3592                         'name': sub_name,
3593                     })
3594
3595             subtitles, automatic_captions = {}, {}
3596             for lang_code, caption_track in captions.items():
3597                 base_url = caption_track.get('baseUrl')
3598                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3599                 if not base_url:
3600                     continue
3601                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3602                 if caption_track.get('kind') != 'asr':
3603                     if not lang_code:
3604                         continue
3605                     process_language(
3606                         subtitles, base_url, lang_code, lang_name, {})
3607                     if not caption_track.get('isTranslatable'):
3608                         continue
3609                 for trans_code, trans_name in translation_languages.items():
3610                     if not trans_code:
3611                         continue
3612                     orig_trans_code = trans_code
3613                     if caption_track.get('kind') != 'asr':
3614                         if 'translated_subs' in self._configuration_arg('skip'):
3615                             continue
3616                         trans_code += f'-{lang_code}'
3617                         trans_name += format_field(lang_name, template=' from %s')
3618                     # Add an "-orig" label to the original language so that it can be distinguished.
3619                     # The subs are returned without "-orig" as well for compatibility
3620                     if lang_code == f'a-{orig_trans_code}':
3621                         process_language(
3622                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3623                     # Setting tlang=lang returns damaged subtitles.
3624                     process_language(automatic_captions, base_url, trans_code, trans_name,
3625                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3626             info['automatic_captions'] = automatic_captions
3627             info['subtitles'] = subtitles
3628
3629         parsed_url = compat_urllib_parse_urlparse(url)
3630         for component in [parsed_url.fragment, parsed_url.query]:
3631             query = compat_parse_qs(component)
3632             for k, v in query.items():
3633                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3634                     d_k += '_time'
3635                     if d_k not in info and k in s_ks:
3636                         info[d_k] = parse_duration(query[k][0])
3637
3638         # Youtube Music Auto-generated description
3639         if video_description:
3640             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
3641             if mobj:
3642                 release_year = mobj.group('release_year')
3643                 release_date = mobj.group('release_date')
3644                 if release_date:
3645                     release_date = release_date.replace('-', '')
3646                     if not release_year:
3647                         release_year = release_date[:4]
3648                 info.update({
3649                     'album': mobj.group('album'.strip()),
3650                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3651                     'track': mobj.group('track').strip(),
3652                     'release_date': release_date,
3653                     'release_year': int_or_none(release_year),
3654                 })
3655
3656         initial_data = None
3657         if webpage:
3658             initial_data = self._extract_yt_initial_variable(
3659                 webpage, self._YT_INITIAL_DATA_RE, video_id,
3660                 'yt initial data')
3661         if not initial_data:
3662             query = {'videoId': video_id}
3663             query.update(self._get_checkok_params())
3664             initial_data = self._extract_response(
3665                 item_id=video_id, ep='next', fatal=False,
3666                 ytcfg=master_ytcfg, query=query,
3667                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3668                 note='Downloading initial data API JSON')
3669
3670         try:  # This will error if there is no livechat
3671             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3672         except (KeyError, IndexError, TypeError):
3673             pass
3674         else:
3675             info.setdefault('subtitles', {})['live_chat'] = [{
3676                 'url': f'https://www.youtube.com/watch?v={video_id}',  # url is needed to set cookies
3677                 'video_id': video_id,
3678                 'ext': 'json',
3679                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3680             }]
3681
3682         if initial_data:
3683             info['chapters'] = (
3684                 self._extract_chapters_from_json(initial_data, duration)
3685                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3686                 or self._extract_chapters_from_description(video_description, duration)
3687                 or None)
3688
3689         contents = traverse_obj(
3690             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3691             expected_type=list, default=[])
3692
3693         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3694         if vpir:
3695             stl = vpir.get('superTitleLink')
3696             if stl:
3697                 stl = self._get_text(stl)
3698                 if try_get(
3699                         vpir,
3700                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3701                     info['location'] = stl
3702                 else:
3703                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3704                     if mobj:
3705                         info.update({
3706                             'series': mobj.group(1),
3707                             'season_number': int(mobj.group(2)),
3708                             'episode_number': int(mobj.group(3)),
3709                         })
3710             for tlb in (try_get(
3711                     vpir,
3712                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3713                     list) or []):
3714                 tbr = tlb.get('toggleButtonRenderer') or {}
3715                 for getter, regex in [(
3716                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3717                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3718                             lambda x: x['accessibility'],
3719                             lambda x: x['accessibilityData']['accessibilityData'],
3720                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3721                     label = (try_get(tbr, getter, dict) or {}).get('label')
3722                     if label:
3723                         mobj = re.match(regex, label)
3724                         if mobj:
3725                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3726                             break
3727             sbr_tooltip = try_get(
3728                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3729             if sbr_tooltip:
3730                 like_count, dislike_count = sbr_tooltip.split(' / ')
3731                 info.update({
3732                     'like_count': str_to_int(like_count),
3733                     'dislike_count': str_to_int(dislike_count),
3734                 })
3735         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3736         if vsir:
3737             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3738             info.update({
3739                 'channel': self._get_text(vor, 'title'),
3740                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3741
3742             rows = try_get(
3743                 vsir,
3744                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3745                 list) or []
3746             multiple_songs = False
3747             for row in rows:
3748                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3749                     multiple_songs = True
3750                     break
3751             for row in rows:
3752                 mrr = row.get('metadataRowRenderer') or {}
3753                 mrr_title = mrr.get('title')
3754                 if not mrr_title:
3755                     continue
3756                 mrr_title = self._get_text(mrr, 'title')
3757                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3758                 if mrr_title == 'License':
3759                     info['license'] = mrr_contents_text
3760                 elif not multiple_songs:
3761                     if mrr_title == 'Album':
3762                         info['album'] = mrr_contents_text
3763                     elif mrr_title == 'Artist':
3764                         info['artist'] = mrr_contents_text
3765                     elif mrr_title == 'Song':
3766                         info['track'] = mrr_contents_text
3767
3768         fallbacks = {
3769             'channel': 'uploader',
3770             'channel_id': 'uploader_id',
3771             'channel_url': 'uploader_url',
3772         }
3773
3774         # The upload date for scheduled, live and past live streams / premieres in microformats
3775         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3776         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3777         upload_date = (
3778             unified_strdate(get_first(microformats, 'uploadDate'))
3779             or unified_strdate(search_meta('uploadDate')))
3780         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3781             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3782         info['upload_date'] = upload_date
3783
3784         for to, frm in fallbacks.items():
3785             if not info.get(to):
3786                 info[to] = info.get(frm)
3787
3788         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3789             v = info.get(s_k)
3790             if v:
3791                 info[d_k] = v
3792
3793         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3794         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3795         is_membersonly = None
3796         is_premium = None
3797         if initial_data and is_private is not None:
3798             is_membersonly = False
3799             is_premium = False
3800             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3801             badge_labels = set()
3802             for content in contents:
3803                 if not isinstance(content, dict):
3804                     continue
3805                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3806             for badge_label in badge_labels:
3807                 if badge_label.lower() == 'members only':
3808                     is_membersonly = True
3809                 elif badge_label.lower() == 'premium':
3810                     is_premium = True
3811                 elif badge_label.lower() == 'unlisted':
3812                     is_unlisted = True
3813
3814         info['availability'] = self._availability(
3815             is_private=is_private,
3816             needs_premium=is_premium,
3817             needs_subscription=is_membersonly,
3818             needs_auth=info['age_limit'] >= 18,
3819             is_unlisted=None if is_private is None else is_unlisted)
3820
3821         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3822
3823         self.mark_watched(video_id, player_responses)
3824
3825         return info
3826
3827
3828 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3829
3830     @staticmethod
3831     def passthrough_smuggled_data(func):
3832         def _smuggle(entries, smuggled_data):
3833             for entry in entries:
3834                 # TODO: Convert URL to music.youtube instead.
3835                 # Do we need to passthrough any other smuggled_data?
3836                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3837                 yield entry
3838
3839         @functools.wraps(func)
3840         def wrapper(self, url):
3841             url, smuggled_data = unsmuggle_url(url, {})
3842             if self.is_music_url(url):
3843                 smuggled_data['is_music_url'] = True
3844             info_dict = func(self, url, smuggled_data)
3845             if smuggled_data and info_dict.get('entries'):
3846                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3847             return info_dict
3848         return wrapper
3849
3850     def _extract_channel_id(self, webpage):
3851         channel_id = self._html_search_meta(
3852             'channelId', webpage, 'channel id', default=None)
3853         if channel_id:
3854             return channel_id
3855         channel_url = self._html_search_meta(
3856             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3857              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3858              'twitter:app:url:googleplay'), webpage, 'channel url')
3859         return self._search_regex(
3860             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3861             channel_url, 'channel id')
3862
3863     @staticmethod
3864     def _extract_basic_item_renderer(item):
3865         # Modified from _extract_grid_item_renderer
3866         known_basic_renderers = (
3867             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3868         )
3869         for key, renderer in item.items():
3870             if not isinstance(renderer, dict):
3871                 continue
3872             elif key in known_basic_renderers:
3873                 return renderer
3874             elif key.startswith('grid') and key.endswith('Renderer'):
3875                 return renderer
3876
3877     def _grid_entries(self, grid_renderer):
3878         for item in grid_renderer['items']:
3879             if not isinstance(item, dict):
3880                 continue
3881             renderer = self._extract_basic_item_renderer(item)
3882             if not isinstance(renderer, dict):
3883                 continue
3884             title = self._get_text(renderer, 'title')
3885
3886             # playlist
3887             playlist_id = renderer.get('playlistId')
3888             if playlist_id:
3889                 yield self.url_result(
3890                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3891                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3892                     video_title=title)
3893                 continue
3894             # video
3895             video_id = renderer.get('videoId')
3896             if video_id:
3897                 yield self._extract_video(renderer)
3898                 continue
3899             # channel
3900             channel_id = renderer.get('channelId')
3901             if channel_id:
3902                 yield self.url_result(
3903                     'https://www.youtube.com/channel/%s' % channel_id,
3904                     ie=YoutubeTabIE.ie_key(), video_title=title)
3905                 continue
3906             # generic endpoint URL support
3907             ep_url = urljoin('https://www.youtube.com/', try_get(
3908                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3909                 compat_str))
3910             if ep_url:
3911                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3912                     if ie.suitable(ep_url):
3913                         yield self.url_result(
3914                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3915                         break
3916
3917     def _music_reponsive_list_entry(self, renderer):
3918         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3919         if video_id:
3920             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3921                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3922         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3923         if playlist_id:
3924             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3925             if video_id:
3926                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3927                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3928             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3929                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3930         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3931         if browse_id:
3932             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3933                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3934
3935     def _shelf_entries_from_content(self, shelf_renderer):
3936         content = shelf_renderer.get('content')
3937         if not isinstance(content, dict):
3938             return
3939         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3940         if renderer:
3941             # TODO: add support for nested playlists so each shelf is processed
3942             # as separate playlist
3943             # TODO: this includes only first N items
3944             yield from self._grid_entries(renderer)
3945         renderer = content.get('horizontalListRenderer')
3946         if renderer:
3947             # TODO
3948             pass
3949
3950     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3951         ep = try_get(
3952             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3953             compat_str)
3954         shelf_url = urljoin('https://www.youtube.com', ep)
3955         if shelf_url:
3956             # Skipping links to another channels, note that checking for
3957             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3958             # will not work
3959             if skip_channels and '/channels?' in shelf_url:
3960                 return
3961             title = self._get_text(shelf_renderer, 'title')
3962             yield self.url_result(shelf_url, video_title=title)
3963         # Shelf may not contain shelf URL, fallback to extraction from content
3964         yield from self._shelf_entries_from_content(shelf_renderer)
3965
3966     def _playlist_entries(self, video_list_renderer):
3967         for content in video_list_renderer['contents']:
3968             if not isinstance(content, dict):
3969                 continue
3970             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3971             if not isinstance(renderer, dict):
3972                 continue
3973             video_id = renderer.get('videoId')
3974             if not video_id:
3975                 continue
3976             yield self._extract_video(renderer)
3977
3978     def _rich_entries(self, rich_grid_renderer):
3979         renderer = try_get(
3980             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3981         video_id = renderer.get('videoId')
3982         if not video_id:
3983             return
3984         yield self._extract_video(renderer)
3985
3986     def _video_entry(self, video_renderer):
3987         video_id = video_renderer.get('videoId')
3988         if video_id:
3989             return self._extract_video(video_renderer)
3990
3991     def _hashtag_tile_entry(self, hashtag_tile_renderer):
3992         url = urljoin('https://youtube.com', traverse_obj(
3993             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
3994         if url:
3995             return self.url_result(
3996                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
3997
3998     def _post_thread_entries(self, post_thread_renderer):
3999         post_renderer = try_get(
4000             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4001         if not post_renderer:
4002             return
4003         # video attachment
4004         video_renderer = try_get(
4005             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4006         video_id = video_renderer.get('videoId')
4007         if video_id:
4008             entry = self._extract_video(video_renderer)
4009             if entry:
4010                 yield entry
4011         # playlist attachment
4012         playlist_id = try_get(
4013             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
4014         if playlist_id:
4015             yield self.url_result(
4016                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4017                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4018         # inline video links
4019         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4020         for run in runs:
4021             if not isinstance(run, dict):
4022                 continue
4023             ep_url = try_get(
4024                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
4025             if not ep_url:
4026                 continue
4027             if not YoutubeIE.suitable(ep_url):
4028                 continue
4029             ep_video_id = YoutubeIE._match_id(ep_url)
4030             if video_id == ep_video_id:
4031                 continue
4032             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4033
4034     def _post_thread_continuation_entries(self, post_thread_continuation):
4035         contents = post_thread_continuation.get('contents')
4036         if not isinstance(contents, list):
4037             return
4038         for content in contents:
4039             renderer = content.get('backstagePostThreadRenderer')
4040             if not isinstance(renderer, dict):
4041                 continue
4042             yield from self._post_thread_entries(renderer)
4043
4044     r''' # unused
4045     def _rich_grid_entries(self, contents):
4046         for content in contents:
4047             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4048             if video_renderer:
4049                 entry = self._video_entry(video_renderer)
4050                 if entry:
4051                     yield entry
4052     '''
4053
4054     def _extract_entries(self, parent_renderer, continuation_list):
4055         # continuation_list is modified in-place with continuation_list = [continuation_token]
4056         continuation_list[:] = [None]
4057         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4058         for content in contents:
4059             if not isinstance(content, dict):
4060                 continue
4061             is_renderer = traverse_obj(
4062                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4063                 expected_type=dict)
4064             if not is_renderer:
4065                 renderer = content.get('richItemRenderer')
4066                 if renderer:
4067                     for entry in self._rich_entries(renderer):
4068                         yield entry
4069                     continuation_list[0] = self._extract_continuation(parent_renderer)
4070                 continue
4071             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4072             for isr_content in isr_contents:
4073                 if not isinstance(isr_content, dict):
4074                     continue
4075
4076                 known_renderers = {
4077                     'playlistVideoListRenderer': self._playlist_entries,
4078                     'gridRenderer': self._grid_entries,
4079                     'reelShelfRenderer': self._grid_entries,
4080                     'shelfRenderer': self._shelf_entries,
4081                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4082                     'backstagePostThreadRenderer': self._post_thread_entries,
4083                     'videoRenderer': lambda x: [self._video_entry(x)],
4084                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4085                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4086                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4087                 }
4088                 for key, renderer in isr_content.items():
4089                     if key not in known_renderers:
4090                         continue
4091                     for entry in known_renderers[key](renderer):
4092                         if entry:
4093                             yield entry
4094                     continuation_list[0] = self._extract_continuation(renderer)
4095                     break
4096
4097             if not continuation_list[0]:
4098                 continuation_list[0] = self._extract_continuation(is_renderer)
4099
4100         if not continuation_list[0]:
4101             continuation_list[0] = self._extract_continuation(parent_renderer)
4102
4103     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4104         continuation_list = [None]
4105         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4106         tab_content = try_get(tab, lambda x: x['content'], dict)
4107         if not tab_content:
4108             return
4109         parent_renderer = (
4110             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4111             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4112         yield from extract_entries(parent_renderer)
4113         continuation = continuation_list[0]
4114
4115         for page_num in itertools.count(1):
4116             if not continuation:
4117                 break
4118             headers = self.generate_api_headers(
4119                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4120             response = self._extract_response(
4121                 item_id=f'{item_id} page {page_num}',
4122                 query=continuation, headers=headers, ytcfg=ytcfg,
4123                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4124
4125             if not response:
4126                 break
4127             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4128             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4129             visitor_data = self._extract_visitor_data(response) or visitor_data
4130
4131             known_continuation_renderers = {
4132                 'playlistVideoListContinuation': self._playlist_entries,
4133                 'gridContinuation': self._grid_entries,
4134                 'itemSectionContinuation': self._post_thread_continuation_entries,
4135                 'sectionListContinuation': extract_entries,  # for feeds
4136             }
4137             continuation_contents = try_get(
4138                 response, lambda x: x['continuationContents'], dict) or {}
4139             continuation_renderer = None
4140             for key, value in continuation_contents.items():
4141                 if key not in known_continuation_renderers:
4142                     continue
4143                 continuation_renderer = value
4144                 continuation_list = [None]
4145                 yield from known_continuation_renderers[key](continuation_renderer)
4146                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4147                 break
4148             if continuation_renderer:
4149                 continue
4150
4151             known_renderers = {
4152                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4153                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4154                 'gridVideoRenderer': (self._grid_entries, 'items'),
4155                 'gridChannelRenderer': (self._grid_entries, 'items'),
4156                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4157                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4158                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4159                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4160             }
4161             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4162             continuation_items = try_get(
4163                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4164             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4165             video_items_renderer = None
4166             for key, value in continuation_item.items():
4167                 if key not in known_renderers:
4168                     continue
4169                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4170                 continuation_list = [None]
4171                 yield from known_renderers[key][0](video_items_renderer)
4172                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4173                 break
4174             if video_items_renderer:
4175                 continue
4176             break
4177
4178     @staticmethod
4179     def _extract_selected_tab(tabs, fatal=True):
4180         for tab in tabs:
4181             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4182             if renderer.get('selected') is True:
4183                 return renderer
4184         else:
4185             if fatal:
4186                 raise ExtractorError('Unable to find selected tab')
4187
4188     def _extract_uploader(self, data):
4189         uploader = {}
4190         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4191         owner = try_get(
4192             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4193         if owner:
4194             owner_text = owner.get('text')
4195             uploader['uploader'] = self._search_regex(
4196                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4197             uploader['uploader_id'] = try_get(
4198                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
4199             uploader['uploader_url'] = urljoin(
4200                 'https://www.youtube.com/',
4201                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
4202         return {k: v for k, v in uploader.items() if v is not None}
4203
4204     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4205         playlist_id = title = description = channel_url = channel_name = channel_id = None
4206         tags = []
4207
4208         selected_tab = self._extract_selected_tab(tabs)
4209         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4210         renderer = try_get(
4211             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4212         if renderer:
4213             channel_name = renderer.get('title')
4214             channel_url = renderer.get('channelUrl')
4215             channel_id = renderer.get('externalId')
4216         else:
4217             renderer = try_get(
4218                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4219
4220         if renderer:
4221             title = renderer.get('title')
4222             description = renderer.get('description', '')
4223             playlist_id = channel_id
4224             tags = renderer.get('keywords', '').split()
4225
4226         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4227         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4228         def _get_uncropped(url):
4229             return url_or_none((url or '').split('=')[0] + '=s0')
4230
4231         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4232         if avatar_thumbnails:
4233             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4234             if uncropped_avatar:
4235                 avatar_thumbnails.append({
4236                     'url': uncropped_avatar,
4237                     'id': 'avatar_uncropped',
4238                     'preference': 1
4239                 })
4240
4241         channel_banners = self._extract_thumbnails(
4242             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4243         for banner in channel_banners:
4244             banner['preference'] = -10
4245
4246         if channel_banners:
4247             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4248             if uncropped_banner:
4249                 channel_banners.append({
4250                     'url': uncropped_banner,
4251                     'id': 'banner_uncropped',
4252                     'preference': -5
4253                 })
4254
4255         primary_thumbnails = self._extract_thumbnails(
4256             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4257
4258         if playlist_id is None:
4259             playlist_id = item_id
4260
4261         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4262         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4263         if title is None:
4264             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4265         title += format_field(selected_tab, 'title', ' - %s')
4266         title += format_field(selected_tab, 'expandedText', ' - %s')
4267
4268         metadata = {
4269             'playlist_id': playlist_id,
4270             'playlist_title': title,
4271             'playlist_description': description,
4272             'uploader': channel_name,
4273             'uploader_id': channel_id,
4274             'uploader_url': channel_url,
4275             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4276             'tags': tags,
4277             'view_count': self._get_count(playlist_stats, 1),
4278             'availability': self._extract_availability(data),
4279             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4280             'playlist_count': self._get_count(playlist_stats, 0),
4281             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4282         }
4283         if not channel_id:
4284             metadata.update(self._extract_uploader(data))
4285         metadata.update({
4286             'channel': metadata['uploader'],
4287             'channel_id': metadata['uploader_id'],
4288             'channel_url': metadata['uploader_url']})
4289         return self.playlist_result(
4290             self._entries(
4291                 selected_tab, playlist_id, ytcfg,
4292                 self._extract_account_syncid(ytcfg, data),
4293                 self._extract_visitor_data(data, ytcfg)),
4294             **metadata)
4295
4296     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4297         first_id = last_id = response = None
4298         for page_num in itertools.count(1):
4299             videos = list(self._playlist_entries(playlist))
4300             if not videos:
4301                 return
4302             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4303             if start >= len(videos):
4304                 return
4305             yield from videos[start:]
4306             first_id = first_id or videos[0]['id']
4307             last_id = videos[-1]['id']
4308             watch_endpoint = try_get(
4309                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4310             headers = self.generate_api_headers(
4311                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4312                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4313             query = {
4314                 'playlistId': playlist_id,
4315                 'videoId': watch_endpoint.get('videoId') or last_id,
4316                 'index': watch_endpoint.get('index') or len(videos),
4317                 'params': watch_endpoint.get('params') or 'OAE%3D'
4318             }
4319             response = self._extract_response(
4320                 item_id='%s page %d' % (playlist_id, page_num),
4321                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4322                 check_get_keys='contents'
4323             )
4324             playlist = try_get(
4325                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4326
4327     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4328         title = playlist.get('title') or try_get(
4329             data, lambda x: x['titleText']['simpleText'], compat_str)
4330         playlist_id = playlist.get('playlistId') or item_id
4331
4332         # Delegating everything except mix playlists to regular tab-based playlist URL
4333         playlist_url = urljoin(url, try_get(
4334             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4335             compat_str))
4336
4337         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4338         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4339         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4340
4341         if playlist_url and playlist_url != url and not is_known_unviewable:
4342             return self.url_result(
4343                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4344                 video_title=title)
4345
4346         return self.playlist_result(
4347             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4348             playlist_id=playlist_id, playlist_title=title)
4349
4350     def _extract_availability(self, data):
4351         """
4352         Gets the availability of a given playlist/tab.
4353         Note: Unless YouTube tells us explicitly, we do not assume it is public
4354         @param data: response
4355         """
4356         is_private = is_unlisted = None
4357         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4358         badge_labels = self._extract_badges(renderer)
4359
4360         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4361         privacy_dropdown_entries = try_get(
4362             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4363         for renderer_dict in privacy_dropdown_entries:
4364             is_selected = try_get(
4365                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4366             if not is_selected:
4367                 continue
4368             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4369             if label:
4370                 badge_labels.add(label.lower())
4371                 break
4372
4373         for badge_label in badge_labels:
4374             if badge_label == 'unlisted':
4375                 is_unlisted = True
4376             elif badge_label == 'private':
4377                 is_private = True
4378             elif badge_label == 'public':
4379                 is_unlisted = is_private = False
4380         return self._availability(is_private, False, False, False, is_unlisted)
4381
4382     @staticmethod
4383     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4384         sidebar_renderer = try_get(
4385             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4386         for item in sidebar_renderer:
4387             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4388             if renderer:
4389                 return renderer
4390
4391     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4392         """
4393         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4394         """
4395         browse_id = params = None
4396         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4397         if not renderer:
4398             return
4399         menu_renderer = try_get(
4400             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4401         for menu_item in menu_renderer:
4402             if not isinstance(menu_item, dict):
4403                 continue
4404             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4405             text = try_get(
4406                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4407             if not text or text.lower() != 'show unavailable videos':
4408                 continue
4409             browse_endpoint = try_get(
4410                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4411             browse_id = browse_endpoint.get('browseId')
4412             params = browse_endpoint.get('params')
4413             break
4414
4415         headers = self.generate_api_headers(
4416             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4417             visitor_data=self._extract_visitor_data(data, ytcfg))
4418         query = {
4419             'params': params or 'wgYCCAA=',
4420             'browseId': browse_id or 'VL%s' % item_id
4421         }
4422         return self._extract_response(
4423             item_id=item_id, headers=headers, query=query,
4424             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4425             note='Downloading API JSON with unavailable videos')
4426
4427     @functools.cached_property
4428     def skip_webpage(self):
4429         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4430
4431     def _extract_webpage(self, url, item_id, fatal=True):
4432         retries = self.get_param('extractor_retries', 3)
4433         count = -1
4434         webpage = data = last_error = None
4435         while count < retries:
4436             count += 1
4437             # Sometimes youtube returns a webpage with incomplete ytInitialData
4438             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4439             if last_error:
4440                 self.report_warning('%s. Retrying ...' % last_error)
4441             try:
4442                 webpage = self._download_webpage(
4443                     url, item_id,
4444                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4445                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4446             except ExtractorError as e:
4447                 if isinstance(e.cause, network_exceptions):
4448                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4449                         last_error = error_to_compat_str(e.cause or e.msg)
4450                         if count < retries:
4451                             continue
4452                 if fatal:
4453                     raise
4454                 self.report_warning(error_to_compat_str(e))
4455                 break
4456             else:
4457                 try:
4458                     self._extract_and_report_alerts(data)
4459                 except ExtractorError as e:
4460                     if fatal:
4461                         raise
4462                     self.report_warning(error_to_compat_str(e))
4463                     break
4464
4465                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4466                     break
4467
4468                 last_error = 'Incomplete yt initial data received'
4469                 if count >= retries:
4470                     if fatal:
4471                         raise ExtractorError(last_error)
4472                     self.report_warning(last_error)
4473                     break
4474
4475         return webpage, data
4476
4477     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4478         """Use if failed to extract ytcfg (and data) from initial webpage"""
4479         if not ytcfg and self.is_authenticated:
4480             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4481             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4482                 raise ExtractorError(
4483                     f'{msg}. If you are not downloading private content, or '
4484                     'your cookies are only for the first account and channel,'
4485                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4486                     expected=True)
4487             self.report_warning(msg, only_once=True)
4488
4489     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4490         data = None
4491         if not self.skip_webpage:
4492             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4493             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4494             # Reject webpage data if redirected to home page without explicitly requesting
4495             selected_tab = self._extract_selected_tab(traverse_obj(
4496                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4497             if (url != 'https://www.youtube.com/feed/recommended'
4498                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4499                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4500                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4501                 if fatal:
4502                     raise ExtractorError(msg, expected=True)
4503                 self.report_warning(msg, only_once=True)
4504         if not data:
4505             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4506             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4507         return data, ytcfg
4508
4509     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4510         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4511         resolve_response = self._extract_response(
4512             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4513             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4514         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4515         for ep_key, ep in endpoints.items():
4516             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4517             if params:
4518                 return self._extract_response(
4519                     item_id=item_id, query=params, ep=ep, headers=headers,
4520                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4521                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4522         err_note = 'Failed to resolve url (does the playlist exist?)'
4523         if fatal:
4524             raise ExtractorError(err_note, expected=True)
4525         self.report_warning(err_note, item_id)
4526
4527     _SEARCH_PARAMS = None
4528
4529     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4530         data = {'query': query}
4531         if params is NO_DEFAULT:
4532             params = self._SEARCH_PARAMS
4533         if params:
4534             data['params'] = params
4535
4536         content_keys = (
4537             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4538             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4539             # ytmusic search
4540             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4541             ('continuationContents', ),
4542         )
4543         display_id = f'query "{query}"'
4544         check_get_keys = tuple({keys[0] for keys in content_keys})
4545         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4546         self._report_playlist_authcheck(ytcfg, fatal=False)
4547
4548         continuation_list = [None]
4549         search = None
4550         for page_num in itertools.count(1):
4551             data.update(continuation_list[0] or {})
4552             headers = self.generate_api_headers(
4553                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4554             search = self._extract_response(
4555                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4556                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4557             slr_contents = traverse_obj(search, *content_keys)
4558             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4559             if not continuation_list[0]:
4560                 break
4561
4562
4563 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4564     IE_DESC = 'YouTube Tabs'
4565     _VALID_URL = r'''(?x:
4566         https?://
4567             (?:\w+\.)?
4568             (?:
4569                 youtube(?:kids)?\.com|
4570                 %(invidious)s
4571             )/
4572             (?:
4573                 (?P<channel_type>channel|c|user|browse)/|
4574                 (?P<not_channel>
4575                     feed/|hashtag/|
4576                     (?:playlist|watch)\?.*?\blist=
4577                 )|
4578                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4579             )
4580             (?P<id>[^/?\#&]+)
4581     )''' % {
4582         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4583         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4584     }
4585     IE_NAME = 'youtube:tab'
4586
4587     _TESTS = [{
4588         'note': 'playlists, multipage',
4589         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4590         'playlist_mincount': 94,
4591         'info_dict': {
4592             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4593             'title': 'Igor Kleiner - Playlists',
4594             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4595             'uploader': 'Igor Kleiner',
4596             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4597             'channel': 'Igor Kleiner',
4598             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4599             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4600             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4601             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4602             'channel_follower_count': int
4603         },
4604     }, {
4605         'note': 'playlists, multipage, different order',
4606         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4607         'playlist_mincount': 94,
4608         'info_dict': {
4609             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4610             'title': 'Igor Kleiner - Playlists',
4611             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4612             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4613             'uploader': 'Igor Kleiner',
4614             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4615             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4616             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4617             'channel': 'Igor Kleiner',
4618             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4619             'channel_follower_count': int
4620         },
4621     }, {
4622         'note': 'playlists, series',
4623         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4624         'playlist_mincount': 5,
4625         'info_dict': {
4626             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4627             'title': '3Blue1Brown - Playlists',
4628             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4629             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4630             'uploader': '3Blue1Brown',
4631             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4632             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4633             'channel': '3Blue1Brown',
4634             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4635             'tags': ['Mathematics'],
4636             'channel_follower_count': int
4637         },
4638     }, {
4639         'note': 'playlists, singlepage',
4640         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4641         'playlist_mincount': 4,
4642         'info_dict': {
4643             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4644             'title': 'ThirstForScience - Playlists',
4645             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4646             'uploader': 'ThirstForScience',
4647             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4648             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4649             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4650             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4651             'tags': 'count:13',
4652             'channel': 'ThirstForScience',
4653             'channel_follower_count': int
4654         }
4655     }, {
4656         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4657         'only_matching': True,
4658     }, {
4659         'note': 'basic, single video playlist',
4660         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4661         'info_dict': {
4662             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4663             'uploader': 'Sergey M.',
4664             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4665             'title': 'youtube-dl public playlist',
4666             'description': '',
4667             'tags': [],
4668             'view_count': int,
4669             'modified_date': '20201130',
4670             'channel': 'Sergey M.',
4671             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4672             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4673             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4674         },
4675         'playlist_count': 1,
4676     }, {
4677         'note': 'empty playlist',
4678         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4679         'info_dict': {
4680             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4681             'uploader': 'Sergey M.',
4682             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4683             'title': 'youtube-dl empty playlist',
4684             'tags': [],
4685             'channel': 'Sergey M.',
4686             'description': '',
4687             'modified_date': '20160902',
4688             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4689             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4690             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4691         },
4692         'playlist_count': 0,
4693     }, {
4694         'note': 'Home tab',
4695         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4696         'info_dict': {
4697             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4698             'title': 'lex will - Home',
4699             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4700             'uploader': 'lex will',
4701             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4702             'channel': 'lex will',
4703             'tags': ['bible', 'history', 'prophesy'],
4704             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4705             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4706             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4707             'channel_follower_count': int
4708         },
4709         'playlist_mincount': 2,
4710     }, {
4711         'note': 'Videos tab',
4712         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4713         'info_dict': {
4714             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4715             'title': 'lex will - Videos',
4716             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4717             'uploader': 'lex will',
4718             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4719             'tags': ['bible', 'history', 'prophesy'],
4720             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4721             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4722             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4723             'channel': 'lex will',
4724             'channel_follower_count': int
4725         },
4726         'playlist_mincount': 975,
4727     }, {
4728         'note': 'Videos tab, sorted by popular',
4729         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4730         'info_dict': {
4731             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4732             'title': 'lex will - Videos',
4733             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4734             'uploader': 'lex will',
4735             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4736             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4737             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4738             'channel': 'lex will',
4739             'tags': ['bible', 'history', 'prophesy'],
4740             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4741             'channel_follower_count': int
4742         },
4743         'playlist_mincount': 199,
4744     }, {
4745         'note': 'Playlists tab',
4746         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4747         'info_dict': {
4748             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4749             'title': 'lex will - Playlists',
4750             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4751             'uploader': 'lex will',
4752             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4753             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4754             'channel': 'lex will',
4755             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4756             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4757             'tags': ['bible', 'history', 'prophesy'],
4758             'channel_follower_count': int
4759         },
4760         'playlist_mincount': 17,
4761     }, {
4762         'note': 'Community tab',
4763         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4764         'info_dict': {
4765             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4766             'title': 'lex will - Community',
4767             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4768             'uploader': 'lex will',
4769             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4770             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4771             'channel': 'lex will',
4772             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4773             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4774             'tags': ['bible', 'history', 'prophesy'],
4775             'channel_follower_count': int
4776         },
4777         'playlist_mincount': 18,
4778     }, {
4779         'note': 'Channels tab',
4780         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4781         'info_dict': {
4782             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4783             'title': 'lex will - Channels',
4784             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4785             'uploader': 'lex will',
4786             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4787             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4788             'channel': 'lex will',
4789             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4790             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4791             'tags': ['bible', 'history', 'prophesy'],
4792             'channel_follower_count': int
4793         },
4794         'playlist_mincount': 12,
4795     }, {
4796         'note': 'Search tab',
4797         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4798         'playlist_mincount': 40,
4799         'info_dict': {
4800             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4801             'title': '3Blue1Brown - Search - linear algebra',
4802             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4803             'uploader': '3Blue1Brown',
4804             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4805             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4806             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4807             'tags': ['Mathematics'],
4808             'channel': '3Blue1Brown',
4809             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4810             'channel_follower_count': int
4811         },
4812     }, {
4813         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4814         'only_matching': True,
4815     }, {
4816         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4817         'only_matching': True,
4818     }, {
4819         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4820         'only_matching': True,
4821     }, {
4822         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4823         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4824         'info_dict': {
4825             'title': '29C3: Not my department',
4826             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4827             'uploader': 'Christiaan008',
4828             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4829             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4830             'tags': [],
4831             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4832             'view_count': int,
4833             'modified_date': '20150605',
4834             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4835             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4836             'channel': 'Christiaan008',
4837         },
4838         'playlist_count': 96,
4839     }, {
4840         'note': 'Large playlist',
4841         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4842         'info_dict': {
4843             'title': 'Uploads from Cauchemar',
4844             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4845             'uploader': 'Cauchemar',
4846             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4847             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4848             'tags': [],
4849             'modified_date': r're:\d{8}',
4850             'channel': 'Cauchemar',
4851             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4852             'view_count': int,
4853             'description': '',
4854             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4855         },
4856         'playlist_mincount': 1123,
4857         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4858     }, {
4859         'note': 'even larger playlist, 8832 videos',
4860         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4861         'only_matching': True,
4862     }, {
4863         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4864         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4865         'info_dict': {
4866             'title': 'Uploads from Interstellar Movie',
4867             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4868             'uploader': 'Interstellar Movie',
4869             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4870             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4871             'tags': [],
4872             'view_count': int,
4873             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4874             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4875             'channel': 'Interstellar Movie',
4876             'description': '',
4877             'modified_date': r're:\d{8}',
4878         },
4879         'playlist_mincount': 21,
4880     }, {
4881         'note': 'Playlist with "show unavailable videos" button',
4882         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4883         'info_dict': {
4884             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4885             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4886             'uploader': 'Phim Siêu Nhân Nhật Bản',
4887             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4888             'view_count': int,
4889             'channel': 'Phim Siêu Nhân Nhật Bản',
4890             'tags': [],
4891             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4892             'description': '',
4893             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4894             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4895             'modified_date': r're:\d{8}',
4896         },
4897         'playlist_mincount': 200,
4898         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4899     }, {
4900         'note': 'Playlist with unavailable videos in page 7',
4901         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4902         'info_dict': {
4903             'title': 'Uploads from BlankTV',
4904             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4905             'uploader': 'BlankTV',
4906             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4907             'channel': 'BlankTV',
4908             'channel_url': 'https://www.youtube.com/c/blanktv',
4909             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4910             'view_count': int,
4911             'tags': [],
4912             'uploader_url': 'https://www.youtube.com/c/blanktv',
4913             'modified_date': r're:\d{8}',
4914             'description': '',
4915         },
4916         'playlist_mincount': 1000,
4917         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4918     }, {
4919         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4920         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4921         'info_dict': {
4922             'title': 'Data Analysis with Dr Mike Pound',
4923             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4924             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4925             'uploader': 'Computerphile',
4926             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4927             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4928             'tags': [],
4929             'view_count': int,
4930             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4931             'channel_url': 'https://www.youtube.com/user/Computerphile',
4932             'channel': 'Computerphile',
4933         },
4934         'playlist_mincount': 11,
4935     }, {
4936         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4937         'only_matching': True,
4938     }, {
4939         'note': 'Playlist URL that does not actually serve a playlist',
4940         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4941         'info_dict': {
4942             'id': 'FqZTN594JQw',
4943             'ext': 'webm',
4944             'title': "Smiley's People 01 detective, Adventure Series, Action",
4945             'uploader': 'STREEM',
4946             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4947             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4948             'upload_date': '20150526',
4949             'license': 'Standard YouTube License',
4950             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4951             'categories': ['People & Blogs'],
4952             'tags': list,
4953             'view_count': int,
4954             'like_count': int,
4955         },
4956         'params': {
4957             'skip_download': True,
4958         },
4959         'skip': 'This video is not available.',
4960         'add_ie': [YoutubeIE.ie_key()],
4961     }, {
4962         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
4963         'only_matching': True,
4964     }, {
4965         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
4966         'only_matching': True,
4967     }, {
4968         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
4969         'info_dict': {
4970             'id': 'GgL890LIznQ',  # This will keep changing
4971             'ext': 'mp4',
4972             'title': str,
4973             'uploader': 'Sky News',
4974             'uploader_id': 'skynews',
4975             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
4976             'upload_date': r're:\d{8}',
4977             'description': str,
4978             'categories': ['News & Politics'],
4979             'tags': list,
4980             'like_count': int,
4981             'release_timestamp': 1642502819,
4982             'channel': 'Sky News',
4983             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
4984             'age_limit': 0,
4985             'view_count': int,
4986             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
4987             'playable_in_embed': True,
4988             'release_date': '20220118',
4989             'availability': 'public',
4990             'live_status': 'is_live',
4991             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
4992             'channel_follower_count': int
4993         },
4994         'params': {
4995             'skip_download': True,
4996         },
4997         'expected_warnings': ['Ignoring subtitle tracks found in '],
4998     }, {
4999         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5000         'info_dict': {
5001             'id': 'a48o2S1cPoo',
5002             'ext': 'mp4',
5003             'title': 'The Young Turks - Live Main Show',
5004             'uploader': 'The Young Turks',
5005             'uploader_id': 'TheYoungTurks',
5006             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5007             'upload_date': '20150715',
5008             'license': 'Standard YouTube License',
5009             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5010             'categories': ['News & Politics'],
5011             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5012             'like_count': int,
5013         },
5014         'params': {
5015             'skip_download': True,
5016         },
5017         'only_matching': True,
5018     }, {
5019         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5020         'only_matching': True,
5021     }, {
5022         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5023         'only_matching': True,
5024     }, {
5025         'note': 'A channel that is not live. Should raise error',
5026         'url': 'https://www.youtube.com/user/numberphile/live',
5027         'only_matching': True,
5028     }, {
5029         'url': 'https://www.youtube.com/feed/trending',
5030         'only_matching': True,
5031     }, {
5032         'url': 'https://www.youtube.com/feed/library',
5033         'only_matching': True,
5034     }, {
5035         'url': 'https://www.youtube.com/feed/history',
5036         'only_matching': True,
5037     }, {
5038         'url': 'https://www.youtube.com/feed/subscriptions',
5039         'only_matching': True,
5040     }, {
5041         'url': 'https://www.youtube.com/feed/watch_later',
5042         'only_matching': True,
5043     }, {
5044         'note': 'Recommended - redirects to home page.',
5045         'url': 'https://www.youtube.com/feed/recommended',
5046         'only_matching': True,
5047     }, {
5048         'note': 'inline playlist with not always working continuations',
5049         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5050         'only_matching': True,
5051     }, {
5052         'url': 'https://www.youtube.com/course',
5053         'only_matching': True,
5054     }, {
5055         'url': 'https://www.youtube.com/zsecurity',
5056         'only_matching': True,
5057     }, {
5058         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5059         'only_matching': True,
5060     }, {
5061         'url': 'https://www.youtube.com/TheYoungTurks/live',
5062         'only_matching': True,
5063     }, {
5064         'url': 'https://www.youtube.com/hashtag/cctv9',
5065         'info_dict': {
5066             'id': 'cctv9',
5067             'title': '#cctv9',
5068             'tags': [],
5069         },
5070         'playlist_mincount': 350,
5071     }, {
5072         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5073         'only_matching': True,
5074     }, {
5075         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5076         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5077         'only_matching': True
5078     }, {
5079         'note': '/browse/ should redirect to /channel/',
5080         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5081         'only_matching': True
5082     }, {
5083         'note': 'VLPL, should redirect to playlist?list=PL...',
5084         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5085         'info_dict': {
5086             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5087             'uploader': 'NoCopyrightSounds',
5088             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5089             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5090             'title': 'NCS Releases',
5091             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5092             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5093             'modified_date': r're:\d{8}',
5094             'view_count': int,
5095             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5096             'tags': [],
5097             'channel': 'NoCopyrightSounds',
5098         },
5099         'playlist_mincount': 166,
5100         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5101     }, {
5102         'note': 'Topic, should redirect to playlist?list=UU...',
5103         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5104         'info_dict': {
5105             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5106             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5107             'title': 'Uploads from Royalty Free Music - Topic',
5108             'uploader': 'Royalty Free Music - Topic',
5109             'tags': [],
5110             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5111             'channel': 'Royalty Free Music - Topic',
5112             'view_count': int,
5113             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5114             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5115             'modified_date': r're:\d{8}',
5116             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5117             'description': '',
5118         },
5119         'expected_warnings': [
5120             'The URL does not have a videos tab',
5121             r'[Uu]navailable videos (are|will be) hidden',
5122         ],
5123         'playlist_mincount': 101,
5124     }, {
5125         'note': 'Topic without a UU playlist',
5126         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5127         'info_dict': {
5128             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5129             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5130             'tags': [],
5131         },
5132         'expected_warnings': [
5133             'the playlist redirect gave error',
5134         ],
5135         'playlist_mincount': 9,
5136     }, {
5137         'note': 'Youtube music Album',
5138         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5139         'info_dict': {
5140             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5141             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5142             'tags': [],
5143             'view_count': int,
5144             'description': '',
5145             'availability': 'unlisted',
5146             'modified_date': r're:\d{8}',
5147         },
5148         'playlist_count': 50,
5149     }, {
5150         'note': 'unlisted single video playlist',
5151         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5152         'info_dict': {
5153             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5154             'uploader': 'colethedj',
5155             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5156             'title': 'yt-dlp unlisted playlist test',
5157             'availability': 'unlisted',
5158             'tags': [],
5159             'modified_date': '20211208',
5160             'channel': 'colethedj',
5161             'view_count': int,
5162             'description': '',
5163             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5164             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5165             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5166         },
5167         'playlist_count': 1,
5168     }, {
5169         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5170         'url': 'https://www.youtube.com/feed/recommended',
5171         'info_dict': {
5172             'id': 'recommended',
5173             'title': 'recommended',
5174             'tags': [],
5175         },
5176         'playlist_mincount': 50,
5177         'params': {
5178             'skip_download': True,
5179             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5180         },
5181     }, {
5182         'note': 'API Fallback: /videos tab, sorted by oldest first',
5183         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5184         'info_dict': {
5185             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5186             'title': 'Cody\'sLab - Videos',
5187             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5188             'uploader': 'Cody\'sLab',
5189             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5190             'channel': 'Cody\'sLab',
5191             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5192             'tags': [],
5193             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5194             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5195             'channel_follower_count': int
5196         },
5197         'playlist_mincount': 650,
5198         'params': {
5199             'skip_download': True,
5200             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5201         },
5202     }, {
5203         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5204         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5205         'info_dict': {
5206             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5207             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5208             'title': 'Uploads from Royalty Free Music - Topic',
5209             'uploader': 'Royalty Free Music - Topic',
5210             'modified_date': r're:\d{8}',
5211             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5212             'description': '',
5213             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5214             'tags': [],
5215             'channel': 'Royalty Free Music - Topic',
5216             'view_count': int,
5217             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5218         },
5219         'expected_warnings': [
5220             'does not have a videos tab',
5221             r'[Uu]navailable videos (are|will be) hidden',
5222         ],
5223         'playlist_mincount': 101,
5224         'params': {
5225             'skip_download': True,
5226             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5227         },
5228     }, {
5229         'note': 'non-standard redirect to regional channel',
5230         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5231         'only_matching': True
5232     }, {
5233         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5234         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5235         'info_dict': {
5236             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5237             'modified_date': '20220407',
5238             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5239             'tags': [],
5240             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5241             'uploader': 'pukkandan',
5242             'availability': 'unlisted',
5243             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5244             'channel': 'pukkandan',
5245             'description': 'Test for collaborative playlist',
5246             'title': 'yt-dlp test - collaborative playlist',
5247             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5248         },
5249         'playlist_mincount': 2
5250     }]
5251
5252     @classmethod
5253     def suitable(cls, url):
5254         return False if YoutubeIE.suitable(url) else super().suitable(url)
5255
5256     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5257
5258     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5259     def _real_extract(self, url, smuggled_data):
5260         item_id = self._match_id(url)
5261         url = compat_urlparse.urlunparse(
5262             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
5263         compat_opts = self.get_param('compat_opts', [])
5264
5265         def get_mobj(url):
5266             mobj = self._URL_RE.match(url).groupdict()
5267             mobj.update((k, '') for k, v in mobj.items() if v is None)
5268             return mobj
5269
5270         mobj, redirect_warning = get_mobj(url), None
5271         # Youtube returns incomplete data if tabname is not lower case
5272         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5273         if is_channel:
5274             if smuggled_data.get('is_music_url'):
5275                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5276                     item_id = item_id[2:]
5277                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5278                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5279                     mdata = self._extract_tab_endpoint(
5280                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5281                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5282                                         get_all=False, expected_type=compat_str)
5283                     if not murl:
5284                         raise ExtractorError('Failed to resolve album to playlist')
5285                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5286                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5287                     pre = f'https://www.youtube.com/channel/{item_id}'
5288
5289         original_tab_name = tab
5290         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5291             # Home URLs should redirect to /videos/
5292             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5293                                 'To download only the videos in the home page, add a "/featured" to the URL')
5294             tab = '/videos'
5295
5296         url = ''.join((pre, tab, post))
5297         mobj = get_mobj(url)
5298
5299         # Handle both video/playlist URLs
5300         qs = parse_qs(url)
5301         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5302
5303         if not video_id and mobj['not_channel'].startswith('watch'):
5304             if not playlist_id:
5305                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5306                 raise ExtractorError('Unable to recognize tab page')
5307             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5308             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5309             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5310             mobj = get_mobj(url)
5311
5312         if video_id and playlist_id:
5313             if self.get_param('noplaylist'):
5314                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5315                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5316                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5317             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5318
5319         data, ytcfg = self._extract_data(url, item_id)
5320
5321         # YouTube may provide a non-standard redirect to the regional channel
5322         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5323         redirect_url = traverse_obj(
5324             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5325         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5326             redirect_url = ''.join((
5327                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5328             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5329             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5330
5331         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5332         if tabs:
5333             selected_tab = self._extract_selected_tab(tabs)
5334             selected_tab_name = selected_tab.get('title', '').lower()
5335             if selected_tab_name == 'home':
5336                 selected_tab_name = 'featured'
5337             requested_tab_name = mobj['tab'][1:]
5338             if 'no-youtube-channel-redirect' not in compat_opts:
5339                 if requested_tab_name == 'live':
5340                     # Live tab should have redirected to the video
5341                     raise ExtractorError('The channel is not currently live', expected=True)
5342                 if requested_tab_name not in ('', selected_tab_name):
5343                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5344                     if not original_tab_name:
5345                         if item_id[:2] == 'UC':
5346                             # Topic channels don't have /videos. Use the equivalent playlist instead
5347                             pl_id = f'UU{item_id[2:]}'
5348                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5349                             try:
5350                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5351                             except ExtractorError:
5352                                 redirect_warning += ' and the playlist redirect gave error'
5353                             else:
5354                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5355                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5356                         if selected_tab_name and selected_tab_name != requested_tab_name:
5357                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5358                     else:
5359                         raise ExtractorError(redirect_warning, expected=True)
5360
5361         if redirect_warning:
5362             self.to_screen(redirect_warning)
5363         self.write_debug(f'Final URL: {url}')
5364
5365         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5366         if 'no-youtube-unavailable-videos' not in compat_opts:
5367             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5368         self._extract_and_report_alerts(data, only_once=True)
5369         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5370         if tabs:
5371             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5372
5373         playlist = traverse_obj(
5374             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5375         if playlist:
5376             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5377
5378         video_id = traverse_obj(
5379             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5380         if video_id:
5381             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5382                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5383             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5384                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5385
5386         raise ExtractorError('Unable to recognize tab page')
5387
5388
5389 class YoutubePlaylistIE(InfoExtractor):
5390     IE_DESC = 'YouTube playlists'
5391     _VALID_URL = r'''(?x)(?:
5392                         (?:https?://)?
5393                         (?:\w+\.)?
5394                         (?:
5395                             (?:
5396                                 youtube(?:kids)?\.com|
5397                                 %(invidious)s
5398                             )
5399                             /.*?\?.*?\blist=
5400                         )?
5401                         (?P<id>%(playlist_id)s)
5402                      )''' % {
5403         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5404         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5405     }
5406     IE_NAME = 'youtube:playlist'
5407     _TESTS = [{
5408         'note': 'issue #673',
5409         'url': 'PLBB231211A4F62143',
5410         'info_dict': {
5411             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5412             'id': 'PLBB231211A4F62143',
5413             'uploader': 'Wickman',
5414             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5415             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5416             'view_count': int,
5417             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5418             'modified_date': r're:\d{8}',
5419             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5420             'channel': 'Wickman',
5421             'tags': [],
5422             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5423         },
5424         'playlist_mincount': 29,
5425     }, {
5426         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5427         'info_dict': {
5428             'title': 'YDL_safe_search',
5429             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5430         },
5431         'playlist_count': 2,
5432         'skip': 'This playlist is private',
5433     }, {
5434         'note': 'embedded',
5435         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5436         'playlist_count': 4,
5437         'info_dict': {
5438             'title': 'JODA15',
5439             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5440             'uploader': 'milan',
5441             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5442             'description': '',
5443             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5444             'tags': [],
5445             'modified_date': '20140919',
5446             'view_count': int,
5447             'channel': 'milan',
5448             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5449             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5450         },
5451         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5452     }, {
5453         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5454         'playlist_mincount': 654,
5455         'info_dict': {
5456             'title': '2018 Chinese New Singles (11/6 updated)',
5457             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5458             'uploader': 'LBK',
5459             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5460             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5461             'channel': 'LBK',
5462             'view_count': int,
5463             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5464             'tags': [],
5465             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5466             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5467             'modified_date': r're:\d{8}',
5468         },
5469         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5470     }, {
5471         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5472         'only_matching': True,
5473     }, {
5474         # music album playlist
5475         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5476         'only_matching': True,
5477     }]
5478
5479     @classmethod
5480     def suitable(cls, url):
5481         if YoutubeTabIE.suitable(url):
5482             return False
5483         from ..utils import parse_qs
5484         qs = parse_qs(url)
5485         if qs.get('v', [None])[0]:
5486             return False
5487         return super().suitable(url)
5488
5489     def _real_extract(self, url):
5490         playlist_id = self._match_id(url)
5491         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5492         url = update_url_query(
5493             'https://www.youtube.com/playlist',
5494             parse_qs(url) or {'list': playlist_id})
5495         if is_music_url:
5496             url = smuggle_url(url, {'is_music_url': True})
5497         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5498
5499
5500 class YoutubeYtBeIE(InfoExtractor):
5501     IE_DESC = 'youtu.be'
5502     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5503     _TESTS = [{
5504         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5505         'info_dict': {
5506             'id': 'yeWKywCrFtk',
5507             'ext': 'mp4',
5508             'title': 'Small Scale Baler and Braiding Rugs',
5509             'uploader': 'Backus-Page House Museum',
5510             'uploader_id': 'backuspagemuseum',
5511             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5512             'upload_date': '20161008',
5513             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5514             'categories': ['Nonprofits & Activism'],
5515             'tags': list,
5516             'like_count': int,
5517             'age_limit': 0,
5518             'playable_in_embed': True,
5519             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5520             'channel': 'Backus-Page House Museum',
5521             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5522             'live_status': 'not_live',
5523             'view_count': int,
5524             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5525             'availability': 'public',
5526             'duration': 59,
5527         },
5528         'params': {
5529             'noplaylist': True,
5530             'skip_download': True,
5531         },
5532     }, {
5533         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5534         'only_matching': True,
5535     }]
5536
5537     def _real_extract(self, url):
5538         mobj = self._match_valid_url(url)
5539         video_id = mobj.group('id')
5540         playlist_id = mobj.group('playlist_id')
5541         return self.url_result(
5542             update_url_query('https://www.youtube.com/watch', {
5543                 'v': video_id,
5544                 'list': playlist_id,
5545                 'feature': 'youtu.be',
5546             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5547
5548
5549 class YoutubeLivestreamEmbedIE(InfoExtractor):
5550     IE_DESC = 'YouTube livestream embeds'
5551     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5552     _TESTS = [{
5553         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5554         'only_matching': True,
5555     }]
5556
5557     def _real_extract(self, url):
5558         channel_id = self._match_id(url)
5559         return self.url_result(
5560             f'https://www.youtube.com/channel/{channel_id}/live',
5561             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5562
5563
5564 class YoutubeYtUserIE(InfoExtractor):
5565     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5566     IE_NAME = 'youtube:user'
5567     _VALID_URL = r'ytuser:(?P<id>.+)'
5568     _TESTS = [{
5569         'url': 'ytuser:phihag',
5570         'only_matching': True,
5571     }]
5572
5573     def _real_extract(self, url):
5574         user_id = self._match_id(url)
5575         return self.url_result(
5576             'https://www.youtube.com/user/%s/videos' % user_id,
5577             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5578
5579
5580 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5581     IE_NAME = 'youtube:favorites'
5582     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5583     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5584     _LOGIN_REQUIRED = True
5585     _TESTS = [{
5586         'url': ':ytfav',
5587         'only_matching': True,
5588     }, {
5589         'url': ':ytfavorites',
5590         'only_matching': True,
5591     }]
5592
5593     def _real_extract(self, url):
5594         return self.url_result(
5595             'https://www.youtube.com/playlist?list=LL',
5596             ie=YoutubeTabIE.ie_key())
5597
5598
5599 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5600     IE_NAME = 'youtube:notif'
5601     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5602     _VALID_URL = r':ytnotif(?:ication)?s?'
5603     _LOGIN_REQUIRED = True
5604     _TESTS = [{
5605         'url': ':ytnotif',
5606         'only_matching': True,
5607     }, {
5608         'url': ':ytnotifications',
5609         'only_matching': True,
5610     }]
5611
5612     def _extract_notification_menu(self, response, continuation_list):
5613         notification_list = traverse_obj(
5614             response,
5615             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5616             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5617             expected_type=list) or []
5618         continuation_list[0] = None
5619         for item in notification_list:
5620             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5621             if entry:
5622                 yield entry
5623             continuation = item.get('continuationItemRenderer')
5624             if continuation:
5625                 continuation_list[0] = continuation
5626
5627     def _extract_notification_renderer(self, notification):
5628         video_id = traverse_obj(
5629             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5630         url = f'https://www.youtube.com/watch?v={video_id}'
5631         channel_id = None
5632         if not video_id:
5633             browse_ep = traverse_obj(
5634                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5635             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5636             post_id = self._search_regex(
5637                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5638                 'post id', default=None)
5639             if not channel_id or not post_id:
5640                 return
5641             # The direct /post url redirects to this in the browser
5642             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5643
5644         channel = traverse_obj(
5645             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5646             expected_type=str)
5647         title = self._search_regex(
5648             rf'{re.escape(channel)} [^:]+: (.+)', self._get_text(notification, 'shortMessage'),
5649             'video title', default=None)
5650         if title:
5651             title = title.replace('\xad', '')  # remove soft hyphens
5652         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5653                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5654                        else None)
5655         return {
5656             '_type': 'url',
5657             'url': url,
5658             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5659             'video_id': video_id,
5660             'title': title,
5661             'channel_id': channel_id,
5662             'channel': channel,
5663             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5664             'upload_date': upload_date,
5665         }
5666
5667     def _notification_menu_entries(self, ytcfg):
5668         continuation_list = [None]
5669         response = None
5670         for page in itertools.count(1):
5671             ctoken = traverse_obj(
5672                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5673             response = self._extract_response(
5674                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5675                 ep='notification/get_notification_menu', check_get_keys='actions',
5676                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5677             yield from self._extract_notification_menu(response, continuation_list)
5678             if not continuation_list[0]:
5679                 break
5680
5681     def _real_extract(self, url):
5682         display_id = 'notifications'
5683         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5684         self._report_playlist_authcheck(ytcfg)
5685         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5686
5687
5688 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5689     IE_DESC = 'YouTube search'
5690     IE_NAME = 'youtube:search'
5691     _SEARCH_KEY = 'ytsearch'
5692     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5693     _TESTS = [{
5694         'url': 'ytsearch5:youtube-dl test video',
5695         'playlist_count': 5,
5696         'info_dict': {
5697             'id': 'youtube-dl test video',
5698             'title': 'youtube-dl test video',
5699         }
5700     }]
5701
5702
5703 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5704     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5705     _SEARCH_KEY = 'ytsearchdate'
5706     IE_DESC = 'YouTube search, newest videos first'
5707     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5708     _TESTS = [{
5709         'url': 'ytsearchdate5:youtube-dl test video',
5710         'playlist_count': 5,
5711         'info_dict': {
5712             'id': 'youtube-dl test video',
5713             'title': 'youtube-dl test video',
5714         }
5715     }]
5716
5717
5718 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5719     IE_DESC = 'YouTube search URLs with sorting and filter support'
5720     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5721     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5722     _TESTS = [{
5723         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5724         'playlist_mincount': 5,
5725         'info_dict': {
5726             'id': 'youtube-dl test video',
5727             'title': 'youtube-dl test video',
5728         }
5729     }, {
5730         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5731         'playlist_mincount': 5,
5732         'info_dict': {
5733             'id': 'python',
5734             'title': 'python',
5735         }
5736     }, {
5737         'url': 'https://www.youtube.com/results?search_query=%23cats',
5738         'playlist_mincount': 1,
5739         'info_dict': {
5740             'id': '#cats',
5741             'title': '#cats',
5742             'entries': [{
5743                 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5744                 'title': '#cats',
5745             }],
5746         },
5747     }, {
5748         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5749         'only_matching': True,
5750     }]
5751
5752     def _real_extract(self, url):
5753         qs = parse_qs(url)
5754         query = (qs.get('search_query') or qs.get('q'))[0]
5755         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5756
5757
5758 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5759     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5760     IE_NAME = 'youtube:music:search_url'
5761     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5762     _TESTS = [{
5763         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5764         'playlist_count': 16,
5765         'info_dict': {
5766             'id': 'royalty free music',
5767             'title': 'royalty free music',
5768         }
5769     }, {
5770         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5771         'playlist_mincount': 30,
5772         'info_dict': {
5773             'id': 'royalty free music - songs',
5774             'title': 'royalty free music - songs',
5775         },
5776         'params': {'extract_flat': 'in_playlist'}
5777     }, {
5778         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5779         'playlist_mincount': 30,
5780         'info_dict': {
5781             'id': 'royalty free music - community playlists',
5782             'title': 'royalty free music - community playlists',
5783         },
5784         'params': {'extract_flat': 'in_playlist'}
5785     }]
5786
5787     _SECTIONS = {
5788         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5789         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5790         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5791         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5792         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5793         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5794     }
5795
5796     def _real_extract(self, url):
5797         qs = parse_qs(url)
5798         query = (qs.get('search_query') or qs.get('q'))[0]
5799         params = qs.get('sp', (None,))[0]
5800         if params:
5801             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5802         else:
5803             section = compat_urllib_parse_unquote_plus((url.split('#') + [''])[1]).lower()
5804             params = self._SECTIONS.get(section)
5805             if not params:
5806                 section = None
5807         title = join_nonempty(query, section, delim=' - ')
5808         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5809
5810
5811 class YoutubeFeedsInfoExtractor(InfoExtractor):
5812     """
5813     Base class for feed extractors
5814     Subclasses must re-define the _FEED_NAME property.
5815     """
5816     _LOGIN_REQUIRED = True
5817     _FEED_NAME = 'feeds'
5818
5819     def _real_initialize(self):
5820         YoutubeBaseInfoExtractor._check_login_required(self)
5821
5822     @classproperty
5823     def IE_NAME(self):
5824         return f'youtube:{self._FEED_NAME}'
5825
5826     def _real_extract(self, url):
5827         return self.url_result(
5828             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5829
5830
5831 class YoutubeWatchLaterIE(InfoExtractor):
5832     IE_NAME = 'youtube:watchlater'
5833     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5834     _VALID_URL = r':ytwatchlater'
5835     _TESTS = [{
5836         'url': ':ytwatchlater',
5837         'only_matching': True,
5838     }]
5839
5840     def _real_extract(self, url):
5841         return self.url_result(
5842             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5843
5844
5845 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5846     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5847     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5848     _FEED_NAME = 'recommended'
5849     _LOGIN_REQUIRED = False
5850     _TESTS = [{
5851         'url': ':ytrec',
5852         'only_matching': True,
5853     }, {
5854         'url': ':ytrecommended',
5855         'only_matching': True,
5856     }, {
5857         'url': 'https://youtube.com',
5858         'only_matching': True,
5859     }]
5860
5861
5862 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5863     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5864     _VALID_URL = r':ytsub(?:scription)?s?'
5865     _FEED_NAME = 'subscriptions'
5866     _TESTS = [{
5867         'url': ':ytsubs',
5868         'only_matching': True,
5869     }, {
5870         'url': ':ytsubscriptions',
5871         'only_matching': True,
5872     }]
5873
5874
5875 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5876     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5877     _VALID_URL = r':ythis(?:tory)?'
5878     _FEED_NAME = 'history'
5879     _TESTS = [{
5880         'url': ':ythistory',
5881         'only_matching': True,
5882     }]
5883
5884
5885 class YoutubeStoriesIE(InfoExtractor):
5886     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5887     IE_NAME = 'youtube:stories'
5888     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5889     _TESTS = [{
5890         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5891         'only_matching': True,
5892     }]
5893
5894     def _real_extract(self, url):
5895         playlist_id = f'RLTD{self._match_id(url)}'
5896         return self.url_result(
5897             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5898             ie=YoutubeTabIE, video_id=playlist_id)
5899
5900
5901 class YoutubeTruncatedURLIE(InfoExtractor):
5902     IE_NAME = 'youtube:truncated_url'
5903     IE_DESC = False  # Do not list
5904     _VALID_URL = r'''(?x)
5905         (?:https?://)?
5906         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5907         (?:watch\?(?:
5908             feature=[a-z_]+|
5909             annotation_id=annotation_[^&]+|
5910             x-yt-cl=[0-9]+|
5911             hl=[^&]*|
5912             t=[0-9]+
5913         )?
5914         |
5915             attribution_link\?a=[^&]+
5916         )
5917         $
5918     '''
5919
5920     _TESTS = [{
5921         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5922         'only_matching': True,
5923     }, {
5924         'url': 'https://www.youtube.com/watch?',
5925         'only_matching': True,
5926     }, {
5927         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5928         'only_matching': True,
5929     }, {
5930         'url': 'https://www.youtube.com/watch?feature=foo',
5931         'only_matching': True,
5932     }, {
5933         'url': 'https://www.youtube.com/watch?hl=en-GB',
5934         'only_matching': True,
5935     }, {
5936         'url': 'https://www.youtube.com/watch?t=2372',
5937         'only_matching': True,
5938     }]
5939
5940     def _real_extract(self, url):
5941         raise ExtractorError(
5942             'Did you forget to quote the URL? Remember that & is a meta '
5943             'character in most shells, so you want to put the URL in quotes, '
5944             'like  youtube-dl '
5945             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5946             ' or simply  youtube-dl BaW_jenozKc  .',
5947             expected=True)
5948
5949
5950 class YoutubeClipIE(InfoExtractor):
5951     IE_NAME = 'youtube:clip'
5952     IE_DESC = False  # Do not list
5953     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/'
5954
5955     def _real_extract(self, url):
5956         self.report_warning('YouTube clips are not currently supported. The entire video will be downloaded instead')
5957         return self.url_result(url, 'Generic')
5958
5959
5960 class YoutubeTruncatedIDIE(InfoExtractor):
5961     IE_NAME = 'youtube:truncated_id'
5962     IE_DESC = False  # Do not list
5963     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
5964
5965     _TESTS = [{
5966         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
5967         'only_matching': True,
5968     }]
5969
5970     def _real_extract(self, url):
5971         video_id = self._match_id(url)
5972         raise ExtractorError(
5973             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
5974             expected=True)