yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16
  17 from .common import InfoExtractor, SearchInfoExtractor
  18 from ..compat import functools  # isort: split
  19 from ..compat import (
  20     compat_chr,
  21     compat_HTTPError,
  22     compat_parse_qs,
  23     compat_str,
  24     compat_urllib_parse_unquote_plus,
  25     compat_urllib_parse_urlencode,
  26     compat_urllib_parse_urlparse,
  27     compat_urlparse,
  28 )
  29 from ..jsinterp import JSInterpreter
  30 from ..utils import (
  31     NO_DEFAULT,
  32     ExtractorError,
  33     bug_reports_message,
  34     classproperty,
  35     clean_html,
  36     datetime_from_str,
  37     dict_get,
  38     error_to_compat_str,
  39     float_or_none,
  40     format_field,
  41     get_first,
  42     int_or_none,
  43     is_html,
  44     join_nonempty,
  45     js_to_json,
  46     mimetype2ext,
  47     network_exceptions,
  48     orderedSet,
  49     parse_codecs,
  50     parse_count,
  51     parse_duration,
  52     parse_iso8601,
  53     parse_qs,
  54     qualities,
  55     remove_end,
  56     remove_start,
  57     smuggle_url,
  58     str_or_none,
  59     str_to_int,
  60     strftime_or_none,
  61     traverse_obj,
  62     try_get,
  63     unescapeHTML,
  64     unified_strdate,
  65     unified_timestamp,
  66     unsmuggle_url,
  67     update_url_query,
  68     url_or_none,
  69     urljoin,
  70     variadic,
  71 )
  72
  73 # any clients starting with _ cannot be explicity requested by the user
  74 INNERTUBE_CLIENTS = {
  75     'web': {
  76         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  77         'INNERTUBE_CONTEXT': {
  78             'client': {
  79                 'clientName': 'WEB',
  80                 'clientVersion': '2.20211221.00.00',
  81             }
  82         },
  83         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  84     },
  85     'web_embedded': {
  86         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  87         'INNERTUBE_CONTEXT': {
  88             'client': {
  89                 'clientName': 'WEB_EMBEDDED_PLAYER',
  90                 'clientVersion': '1.20211215.00.01',
  91             },
  92         },
  93         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  94     },
  95     'web_music': {
  96         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  97         'INNERTUBE_HOST': 'music.youtube.com',
  98         'INNERTUBE_CONTEXT': {
  99             'client': {
 100                 'clientName': 'WEB_REMIX',
 101                 'clientVersion': '1.20211213.00.00',
 102             }
 103         },
 104         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 105     },
 106     'web_creator': {
 107         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 108         'INNERTUBE_CONTEXT': {
 109             'client': {
 110                 'clientName': 'WEB_CREATOR',
 111                 'clientVersion': '1.20211220.02.00',
 112             }
 113         },
 114         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 115     },
 116     'android': {
 117         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 118         'INNERTUBE_CONTEXT': {
 119             'client': {
 120                 'clientName': 'ANDROID',
 121                 'clientVersion': '16.49',
 122             }
 123         },
 124         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 125         'REQUIRE_JS_PLAYER': False
 126     },
 127     'android_embedded': {
 128         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 129         'INNERTUBE_CONTEXT': {
 130             'client': {
 131                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 132                 'clientVersion': '16.49',
 133             },
 134         },
 135         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 136         'REQUIRE_JS_PLAYER': False
 137     },
 138     'android_music': {
 139         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 140         'INNERTUBE_CONTEXT': {
 141             'client': {
 142                 'clientName': 'ANDROID_MUSIC',
 143                 'clientVersion': '4.57',
 144             }
 145         },
 146         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 147         'REQUIRE_JS_PLAYER': False
 148     },
 149     'android_creator': {
 150         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 151         'INNERTUBE_CONTEXT': {
 152             'client': {
 153                 'clientName': 'ANDROID_CREATOR',
 154                 'clientVersion': '21.47',
 155             },
 156         },
 157         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 158         'REQUIRE_JS_PLAYER': False
 159     },
 160     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 161     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 162     'ios': {
 163         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 164         'INNERTUBE_CONTEXT': {
 165             'client': {
 166                 'clientName': 'IOS',
 167                 'clientVersion': '16.46',
 168                 'deviceModel': 'iPhone14,3',
 169             }
 170         },
 171         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 172         'REQUIRE_JS_PLAYER': False
 173     },
 174     'ios_embedded': {
 175         'INNERTUBE_CONTEXT': {
 176             'client': {
 177                 'clientName': 'IOS_MESSAGES_EXTENSION',
 178                 'clientVersion': '16.46',
 179                 'deviceModel': 'iPhone14,3',
 180             },
 181         },
 182         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 183         'REQUIRE_JS_PLAYER': False
 184     },
 185     'ios_music': {
 186         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 187         'INNERTUBE_CONTEXT': {
 188             'client': {
 189                 'clientName': 'IOS_MUSIC',
 190                 'clientVersion': '4.57',
 191             },
 192         },
 193         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 194         'REQUIRE_JS_PLAYER': False
 195     },
 196     'ios_creator': {
 197         'INNERTUBE_CONTEXT': {
 198             'client': {
 199                 'clientName': 'IOS_CREATOR',
 200                 'clientVersion': '21.47',
 201             },
 202         },
 203         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 204         'REQUIRE_JS_PLAYER': False
 205     },
 206     # mweb has 'ultralow' formats
 207     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 208     'mweb': {
 209         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 210         'INNERTUBE_CONTEXT': {
 211             'client': {
 212                 'clientName': 'MWEB',
 213                 'clientVersion': '2.20211221.01.00',
 214             }
 215         },
 216         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 217     },
 218     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 219     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 220     'tv_embedded': {
 221         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 222         'INNERTUBE_CONTEXT': {
 223             'client': {
 224                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 225                 'clientVersion': '2.0',
 226             },
 227         },
 228         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 229     },
 230 }
 231
 232
 233 def _split_innertube_client(client_name):
 234     variant, *base = client_name.rsplit('.', 1)
 235     if base:
 236         return variant, base[0], variant
 237     base, *variant = client_name.split('_', 1)
 238     return client_name, base, variant[0] if variant else None
 239
 240
 241 def build_innertube_clients():
 242     THIRD_PARTY = {
 243         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 244     }
 245     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 246     priority = qualities(BASE_CLIENTS[::-1])
 247
 248     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 249         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 250         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 251         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 252         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 253
 254         _, base_client, variant = _split_innertube_client(client)
 255         ytcfg['priority'] = 10 * priority(base_client)
 256
 257         if not variant:
 258             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 259             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 260             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 261             embedscreen['priority'] -= 3
 262         elif variant == 'embedded':
 263             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 264             ytcfg['priority'] -= 2
 265         else:
 266             ytcfg['priority'] -= 3
 267
 268
 269 build_innertube_clients()
 270
 271
 272 class YoutubeBaseInfoExtractor(InfoExtractor):
 273     """Provide base functions for Youtube extractors"""
 274
 275     _RESERVED_NAMES = (
 276         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 277         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 278         r'browse|oembed|get_video_info|iframe_api|s/player|'
 279         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 280
 281     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 282
 283     # _NETRC_MACHINE = 'youtube'
 284
 285     # If True it will raise an error if no login info is provided
 286     _LOGIN_REQUIRED = False
 287
 288     _INVIDIOUS_SITES = (
 289         # invidious-redirect websites
 290         r'(?:www\.)?redirect\.invidious\.io',
 291         r'(?:(?:www|dev)\.)?invidio\.us',
 292         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 293         r'(?:www\.)?invidious\.pussthecat\.org',
 294         r'(?:www\.)?invidious\.zee\.li',
 295         r'(?:www\.)?invidious\.ethibox\.fr',
 296         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 297         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 298         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 299         # youtube-dl invidious instances list
 300         r'(?:(?:www|no)\.)?invidiou\.sh',
 301         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 302         r'(?:www\.)?invidious\.kabi\.tk',
 303         r'(?:www\.)?invidious\.mastodon\.host',
 304         r'(?:www\.)?invidious\.zapashcanon\.fr',
 305         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 306         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 307         r'(?:www\.)?invidious\.himiko\.cloud',
 308         r'(?:www\.)?invidious\.reallyancient\.tech',
 309         r'(?:www\.)?invidious\.tube',
 310         r'(?:www\.)?invidiou\.site',
 311         r'(?:www\.)?invidious\.site',
 312         r'(?:www\.)?invidious\.xyz',
 313         r'(?:www\.)?invidious\.nixnet\.xyz',
 314         r'(?:www\.)?invidious\.048596\.xyz',
 315         r'(?:www\.)?invidious\.drycat\.fr',
 316         r'(?:www\.)?inv\.skyn3t\.in',
 317         r'(?:www\.)?tube\.poal\.co',
 318         r'(?:www\.)?tube\.connect\.cafe',
 319         r'(?:www\.)?vid\.wxzm\.sx',
 320         r'(?:www\.)?vid\.mint\.lgbt',
 321         r'(?:www\.)?vid\.puffyan\.us',
 322         r'(?:www\.)?yewtu\.be',
 323         r'(?:www\.)?yt\.elukerio\.org',
 324         r'(?:www\.)?yt\.lelux\.fi',
 325         r'(?:www\.)?invidious\.ggc-project\.de',
 326         r'(?:www\.)?yt\.maisputain\.ovh',
 327         r'(?:www\.)?ytprivate\.com',
 328         r'(?:www\.)?invidious\.13ad\.de',
 329         r'(?:www\.)?invidious\.toot\.koeln',
 330         r'(?:www\.)?invidious\.fdn\.fr',
 331         r'(?:www\.)?watch\.nettohikari\.com',
 332         r'(?:www\.)?invidious\.namazso\.eu',
 333         r'(?:www\.)?invidious\.silkky\.cloud',
 334         r'(?:www\.)?invidious\.exonip\.de',
 335         r'(?:www\.)?invidious\.riverside\.rocks',
 336         r'(?:www\.)?invidious\.blamefran\.net',
 337         r'(?:www\.)?invidious\.moomoo\.de',
 338         r'(?:www\.)?ytb\.trom\.tf',
 339         r'(?:www\.)?yt\.cyberhost\.uk',
 340         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 341         r'(?:www\.)?qklhadlycap4cnod\.onion',
 342         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 343         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 344         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 345         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 346         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 347         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 348         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 349         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 350         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 351         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 352         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 353         r'(?:www\.)?piped\.kavin\.rocks',
 354         r'(?:www\.)?piped\.silkky\.cloud',
 355         r'(?:www\.)?piped\.tokhmi\.xyz',
 356         r'(?:www\.)?piped\.moomoo\.me',
 357         r'(?:www\.)?il\.ax',
 358         r'(?:www\.)?piped\.syncpundit\.com',
 359         r'(?:www\.)?piped\.mha\.fi',
 360         r'(?:www\.)?piped\.mint\.lgbt',
 361         r'(?:www\.)?piped\.privacy\.com\.de',
 362     )
 363
 364     def _initialize_consent(self):
 365         cookies = self._get_cookies('https://www.youtube.com/')
 366         if cookies.get('__Secure-3PSID'):
 367             return
 368         consent_id = None
 369         consent = cookies.get('CONSENT')
 370         if consent:
 371             if 'YES' in consent.value:
 372                 return
 373             consent_id = self._search_regex(
 374                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 375         if not consent_id:
 376             consent_id = random.randint(100, 999)
 377         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 378
 379     def _initialize_pref(self):
 380         cookies = self._get_cookies('https://www.youtube.com/')
 381         pref_cookie = cookies.get('PREF')
 382         pref = {}
 383         if pref_cookie:
 384             try:
 385                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 386             except ValueError:
 387                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 388         pref.update({'hl': 'en', 'tz': 'UTC'})
 389         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 390
 391     def _real_initialize(self):
 392         self._initialize_pref()
 393         self._initialize_consent()
 394         self._check_login_required()
 395
 396     def _check_login_required(self):
 397         if self._LOGIN_REQUIRED and not self._cookies_passed:
 398             self.raise_login_required('Login details are needed to download this content', method='cookies')
 399
 400     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 401     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 402
 403     def _get_default_ytcfg(self, client='web'):
 404         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 405
 406     def _get_innertube_host(self, client='web'):
 407         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 408
 409     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 410         # try_get but with fallback to default ytcfg client values when present
 411         _func = lambda y: try_get(y, getter, expected_type)
 412         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 413
 414     def _extract_client_name(self, ytcfg, default_client='web'):
 415         return self._ytcfg_get_safe(
 416             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 417                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 418
 419     def _extract_client_version(self, ytcfg, default_client='web'):
 420         return self._ytcfg_get_safe(
 421             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 422                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 423
 424     def _select_api_hostname(self, req_api_hostname, default_client=None):
 425         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 426                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 427
 428     def _extract_api_key(self, ytcfg=None, default_client='web'):
 429         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 430
 431     def _extract_context(self, ytcfg=None, default_client='web'):
 432         context = get_first(
 433             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 434         # Enforce language and tz for extraction
 435         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 436         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 437         return context
 438
 439     _SAPISID = None
 440
 441     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 442         time_now = round(time.time())
 443         if self._SAPISID is None:
 444             yt_cookies = self._get_cookies('https://www.youtube.com')
 445             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 446             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 447             sapisid_cookie = dict_get(
 448                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 449             if sapisid_cookie and sapisid_cookie.value:
 450                 self._SAPISID = sapisid_cookie.value
 451                 self.write_debug('Extracted SAPISID cookie')
 452                 # SAPISID cookie is required if not already present
 453                 if not yt_cookies.get('SAPISID'):
 454                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 455                     self._set_cookie(
 456                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 457             else:
 458                 self._SAPISID = False
 459         if not self._SAPISID:
 460             return None
 461         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 462         sapisidhash = hashlib.sha1(
 463             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 464         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 465
 466     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 467                   note='Downloading API JSON', errnote='Unable to download API page',
 468                   context=None, api_key=None, api_hostname=None, default_client='web'):
 469
 470         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 471         data.update(query)
 472         real_headers = self.generate_api_headers(default_client=default_client)
 473         real_headers.update({'content-type': 'application/json'})
 474         if headers:
 475             real_headers.update(headers)
 476         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 477                    or api_key or self._extract_api_key(default_client=default_client))
 478         return self._download_json(
 479             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 480             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 481             data=json.dumps(data).encode('utf8'), headers=real_headers,
 482             query={'key': api_key, 'prettyPrint': 'false'})
 483
 484     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 485         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 486
 487     @staticmethod
 488     def _extract_session_index(*data):
 489         """
 490         Index of current account in account list.
 491         See: https://github.com/yt-dlp/yt-dlp/pull/519
 492         """
 493         for ytcfg in data:
 494             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 495             if session_index is not None:
 496                 return session_index
 497
 498     # Deprecated?
 499     def _extract_identity_token(self, ytcfg=None, webpage=None):
 500         if ytcfg:
 501             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 502             if token:
 503                 return token
 504         if webpage:
 505             return self._search_regex(
 506                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 507                 'identity token', default=None, fatal=False)
 508
 509     @staticmethod
 510     def _extract_account_syncid(*args):
 511         """
 512         Extract syncId required to download private playlists of secondary channels
 513         @params response and/or ytcfg
 514         """
 515         for data in args:
 516             # ytcfg includes channel_syncid if on secondary channel
 517             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 518             if delegated_sid:
 519                 return delegated_sid
 520             sync_ids = (try_get(
 521                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 522                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 523             if len(sync_ids) >= 2 and sync_ids[1]:
 524                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 525                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 526                 return sync_ids[0]
 527
 528     @staticmethod
 529     def _extract_visitor_data(*args):
 530         """
 531         Extracts visitorData from an API response or ytcfg
 532         Appears to be used to track session state
 533         """
 534         return get_first(
 535             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 536             expected_type=str)
 537
 538     @functools.cached_property
 539     def is_authenticated(self):
 540         return bool(self._generate_sapisidhash_header())
 541
 542     def extract_ytcfg(self, video_id, webpage):
 543         if not webpage:
 544             return {}
 545         return self._parse_json(
 546             self._search_regex(
 547                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 548                 default='{}'), video_id, fatal=False) or {}
 549
 550     def generate_api_headers(
 551             self, *, ytcfg=None, account_syncid=None, session_index=None,
 552             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 553
 554         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 555         headers = {
 556             'X-YouTube-Client-Name': compat_str(
 557                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 558             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 559             'Origin': origin,
 560             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 561             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 562             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 563         }
 564         if session_index is None:
 565             session_index = self._extract_session_index(ytcfg)
 566         if account_syncid or session_index is not None:
 567             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 568
 569         auth = self._generate_sapisidhash_header(origin)
 570         if auth is not None:
 571             headers['Authorization'] = auth
 572             headers['X-Origin'] = origin
 573         return {h: v for h, v in headers.items() if v is not None}
 574
 575     def _download_ytcfg(self, client, video_id):
 576         url = {
 577             'web': 'https://www.youtube.com',
 578             'web_music': 'https://music.youtube.com',
 579             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 580         }.get(client)
 581         if not url:
 582             return {}
 583         webpage = self._download_webpage(
 584             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 585         return self.extract_ytcfg(video_id, webpage) or {}
 586
 587     @staticmethod
 588     def _build_api_continuation_query(continuation, ctp=None):
 589         query = {
 590             'continuation': continuation
 591         }
 592         # TODO: Inconsistency with clickTrackingParams.
 593         # Currently we have a fixed ctp contained within context (from ytcfg)
 594         # and a ctp in root query for continuation.
 595         if ctp:
 596             query['clickTracking'] = {'clickTrackingParams': ctp}
 597         return query
 598
 599     @classmethod
 600     def _extract_next_continuation_data(cls, renderer):
 601         next_continuation = try_get(
 602             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 603                        lambda x: x['continuation']['reloadContinuationData']), dict)
 604         if not next_continuation:
 605             return
 606         continuation = next_continuation.get('continuation')
 607         if not continuation:
 608             return
 609         ctp = next_continuation.get('clickTrackingParams')
 610         return cls._build_api_continuation_query(continuation, ctp)
 611
 612     @classmethod
 613     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 614         if isinstance(continuation_ep, dict):
 615             continuation = try_get(
 616                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 617             if not continuation:
 618                 return
 619             ctp = continuation_ep.get('clickTrackingParams')
 620             return cls._build_api_continuation_query(continuation, ctp)
 621
 622     @classmethod
 623     def _extract_continuation(cls, renderer):
 624         next_continuation = cls._extract_next_continuation_data(renderer)
 625         if next_continuation:
 626             return next_continuation
 627
 628         contents = []
 629         for key in ('contents', 'items'):
 630             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 631
 632         for content in contents:
 633             if not isinstance(content, dict):
 634                 continue
 635             continuation_ep = try_get(
 636                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 637                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 638                 dict)
 639             continuation = cls._extract_continuation_ep_data(continuation_ep)
 640             if continuation:
 641                 return continuation
 642
 643     @classmethod
 644     def _extract_alerts(cls, data):
 645         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 646             if not isinstance(alert_dict, dict):
 647                 continue
 648             for alert in alert_dict.values():
 649                 alert_type = alert.get('type')
 650                 if not alert_type:
 651                     continue
 652                 message = cls._get_text(alert, 'text')
 653                 if message:
 654                     yield alert_type, message
 655
 656     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 657         errors = []
 658         warnings = []
 659         for alert_type, alert_message in alerts:
 660             if alert_type.lower() == 'error' and fatal:
 661                 errors.append([alert_type, alert_message])
 662             else:
 663                 warnings.append([alert_type, alert_message])
 664
 665         for alert_type, alert_message in (warnings + errors[:-1]):
 666             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 667         if errors:
 668             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 669
 670     def _extract_and_report_alerts(self, data, *args, **kwargs):
 671         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 672
 673     def _extract_badges(self, renderer: dict):
 674         badges = set()
 675         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 676             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 677             if label:
 678                 badges.add(label.lower())
 679         return badges
 680
 681     @staticmethod
 682     def _get_text(data, *path_list, max_runs=None):
 683         for path in path_list or [None]:
 684             if path is None:
 685                 obj = [data]
 686             else:
 687                 obj = traverse_obj(data, path, default=[])
 688                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 689                     obj = [obj]
 690             for item in obj:
 691                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 692                 if text:
 693                     return text
 694                 runs = try_get(item, lambda x: x['runs'], list) or []
 695                 if not runs and isinstance(item, list):
 696                     runs = item
 697
 698                 runs = runs[:min(len(runs), max_runs or len(runs))]
 699                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 700                 if text:
 701                     return text
 702
 703     def _get_count(self, data, *path_list):
 704         count_text = self._get_text(data, *path_list) or ''
 705         count = parse_count(count_text)
 706         if count is None:
 707             count = str_to_int(
 708                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 709         return count
 710
 711     @staticmethod
 712     def _extract_thumbnails(data, *path_list):
 713         """
 714         Extract thumbnails from thumbnails dict
 715         @param path_list: path list to level that contains 'thumbnails' key
 716         """
 717         thumbnails = []
 718         for path in path_list or [()]:
 719             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 720                 thumbnail_url = url_or_none(thumbnail.get('url'))
 721                 if not thumbnail_url:
 722                     continue
 723                 # Sometimes youtube gives a wrong thumbnail URL. See:
 724                 # https://github.com/yt-dlp/yt-dlp/issues/233
 725                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 726                 if 'maxresdefault' in thumbnail_url:
 727                     thumbnail_url = thumbnail_url.split('?')[0]
 728                 thumbnails.append({
 729                     'url': thumbnail_url,
 730                     'height': int_or_none(thumbnail.get('height')),
 731                     'width': int_or_none(thumbnail.get('width')),
 732                 })
 733         return thumbnails
 734
 735     @staticmethod
 736     def extract_relative_time(relative_time_text):
 737         """
 738         Extracts a relative time from string and converts to dt object
 739         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 740         """
 741         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 742         if mobj:
 743             start = mobj.group('start')
 744             if start:
 745                 return datetime_from_str(start)
 746             try:
 747                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 748             except ValueError:
 749                 return None
 750
 751     def _extract_time_text(self, renderer, *path_list):
 752         """@returns (timestamp, time_text)"""
 753         text = self._get_text(renderer, *path_list) or ''
 754         dt = self.extract_relative_time(text)
 755         timestamp = None
 756         if isinstance(dt, datetime.datetime):
 757             timestamp = calendar.timegm(dt.timetuple())
 758
 759         if timestamp is None:
 760             timestamp = (
 761                 unified_timestamp(text) or unified_timestamp(
 762                     self._search_regex(
 763                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 764                         text.lower(), 'time text', default=None)))
 765
 766         if text and timestamp is None:
 767             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 768         return timestamp, text
 769
 770     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 771                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 772                           default_client='web'):
 773         response = None
 774         last_error = None
 775         count = -1
 776         retries = self.get_param('extractor_retries', 3)
 777         if check_get_keys is None:
 778             check_get_keys = []
 779         while count < retries:
 780             count += 1
 781             if last_error:
 782                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 783             try:
 784                 response = self._call_api(
 785                     ep=ep, fatal=True, headers=headers,
 786                     video_id=item_id, query=query,
 787                     context=self._extract_context(ytcfg, default_client),
 788                     api_key=self._extract_api_key(ytcfg, default_client),
 789                     api_hostname=api_hostname, default_client=default_client,
 790                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 791             except ExtractorError as e:
 792                 if isinstance(e.cause, network_exceptions):
 793                     if isinstance(e.cause, compat_HTTPError):
 794                         first_bytes = e.cause.read(512)
 795                         if not is_html(first_bytes):
 796                             yt_error = try_get(
 797                                 self._parse_json(
 798                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 799                                 lambda x: x['error']['message'], compat_str)
 800                             if yt_error:
 801                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 802                     # Downloading page may result in intermittent 5xx HTTP error
 803                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 804                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 805                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 806                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 807                         last_error = error_to_compat_str(e.cause or e.msg)
 808                         if count < retries:
 809                             continue
 810                 if fatal:
 811                     raise
 812                 else:
 813                     self.report_warning(error_to_compat_str(e))
 814                     return
 815
 816             else:
 817                 try:
 818                     self._extract_and_report_alerts(response, only_once=True)
 819                 except ExtractorError as e:
 820                     # YouTube servers may return errors we want to retry on in a 200 OK response
 821                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 822                     if 'unknown error' in e.msg.lower():
 823                         last_error = e.msg
 824                         continue
 825                     if fatal:
 826                         raise
 827                     self.report_warning(error_to_compat_str(e))
 828                     return
 829                 if not check_get_keys or dict_get(response, check_get_keys):
 830                     break
 831                 # Youtube sometimes sends incomplete data
 832                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 833                 last_error = 'Incomplete data received'
 834                 if count >= retries:
 835                     if fatal:
 836                         raise ExtractorError(last_error)
 837                     else:
 838                         self.report_warning(last_error)
 839                         return
 840         return response
 841
 842     @staticmethod
 843     def is_music_url(url):
 844         return re.match(r'https?://music\.youtube\.com/', url) is not None
 845
 846     def _extract_video(self, renderer):
 847         video_id = renderer.get('videoId')
 848         title = self._get_text(renderer, 'title')
 849         description = self._get_text(renderer, 'descriptionSnippet')
 850         duration = parse_duration(self._get_text(
 851             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 852         if duration is None:
 853             duration = parse_duration(self._search_regex(
 854                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 855                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 856                 video_id, default=None, group='duration'))
 857
 858         view_count = self._get_count(renderer, 'viewCountText')
 859
 860         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 861         channel_id = traverse_obj(
 862             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 863             expected_type=str, get_all=False)
 864         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 865         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 866         overlay_style = traverse_obj(
 867             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 868             get_all=False, expected_type=str)
 869         badges = self._extract_badges(renderer)
 870         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 871         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 872             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 873             expected_type=str)) or ''
 874         url = f'https://www.youtube.com/watch?v={video_id}'
 875         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 876             url = f'https://www.youtube.com/shorts/{video_id}'
 877
 878         return {
 879             '_type': 'url',
 880             'ie_key': YoutubeIE.ie_key(),
 881             'id': video_id,
 882             'url': url,
 883             'title': title,
 884             'description': description,
 885             'duration': duration,
 886             'view_count': view_count,
 887             'uploader': uploader,
 888             'channel_id': channel_id,
 889             'thumbnails': thumbnails,
 890             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 891                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 892                             else None),
 893             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 894                             else 'was_live' if 'streamed' in time_text.lower()
 895                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 896                             else None),
 897             'release_timestamp': scheduled_timestamp,
 898             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 899         }
 900
 901
 902 class YoutubeIE(YoutubeBaseInfoExtractor):
 903     IE_DESC = 'YouTube'
 904     _VALID_URL = r"""(?x)^
 905                      (
 906                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 907                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 908                             (?:www\.)?deturl\.com/www\.youtube\.com|
 909                             (?:www\.)?pwnyoutube\.com|
 910                             (?:www\.)?hooktube\.com|
 911                             (?:www\.)?yourepeat\.com|
 912                             tube\.majestyc\.net|
 913                             %(invidious)s|
 914                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 915                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 916                          (?:                                                  # the various things that can precede the ID:
 917                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 918                              |(?:                                             # or the v= param in all its forms
 919                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 920                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 921                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 922                                  v=
 923                              )
 924                          ))
 925                          |(?:
 926                             youtu\.be|                                        # just youtu.be/xxxx
 927                             vid\.plus|                                        # or vid.plus/xxxx
 928                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 929                             %(invidious)s
 930                          )/
 931                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 932                          )
 933                      )?                                                       # all until now is optional -> you can pass the naked ID
 934                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 935                      (?(1).+)?                                                # if we found the ID, everything can follow
 936                      (?:\#|$)""" % {
 937         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 938     }
 939     _PLAYER_INFO_RE = (
 940         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 941         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 942         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 943     )
 944     _formats = {
 945         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 946         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 947         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 948         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 949         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 950         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 951         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 953         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 954         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 955         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 956         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 957         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 958         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 959         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 960         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 961         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 962         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 963
 964
 965         # 3D videos
 966         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 967         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 968         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 969         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 970         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 971         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 972         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 973
 974         # Apple HTTP Live Streaming
 975         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 976         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 977         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 978         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 979         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 980         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 981         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 982         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 983
 984         # DASH mp4 video
 985         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 986         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 988         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 989         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 990         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 991         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 992         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 993         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 994         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 995         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 996         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 997
 998         # Dash mp4 audio
 999         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1000         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1001         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1002         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1003         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1004         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1005         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1006
1007         # Dash webm
1008         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1009         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1010         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1011         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1012         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1013         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1014         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1015         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1016         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1017         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1018         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1020         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1021         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1022         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1023         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1024         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1025         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1026         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1027         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1028         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1029         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1030
1031         # Dash webm audio
1032         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1033         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1034
1035         # Dash webm audio with opus inside
1036         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1037         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1038         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1039
1040         # RTMP (unnamed)
1041         '_rtmp': {'protocol': 'rtmp'},
1042
1043         # av01 video only formats sometimes served with "unknown" codecs
1044         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1045         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1046         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1047         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1048         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1049         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1050         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1051         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1052     }
1053     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1054
1055     _GEO_BYPASS = False
1056
1057     IE_NAME = 'youtube'
1058     _TESTS = [
1059         {
1060             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1061             'info_dict': {
1062                 'id': 'BaW_jenozKc',
1063                 'ext': 'mp4',
1064                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1065                 'uploader': 'Philipp Hagemeister',
1066                 'uploader_id': 'phihag',
1067                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1068                 'channel': 'Philipp Hagemeister',
1069                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1070                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1071                 'upload_date': '20121002',
1072                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1073                 'categories': ['Science & Technology'],
1074                 'tags': ['youtube-dl'],
1075                 'duration': 10,
1076                 'view_count': int,
1077                 'like_count': int,
1078                 'availability': 'public',
1079                 'playable_in_embed': True,
1080                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1081                 'live_status': 'not_live',
1082                 'age_limit': 0,
1083                 'start_time': 1,
1084                 'end_time': 9,
1085                 'channel_follower_count': int
1086             }
1087         },
1088         {
1089             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1090             'note': 'Embed-only video (#1746)',
1091             'info_dict': {
1092                 'id': 'yZIXLfi8CZQ',
1093                 'ext': 'mp4',
1094                 'upload_date': '20120608',
1095                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1096                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1097                 'uploader': 'SET India',
1098                 'uploader_id': 'setindia',
1099                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1100                 'age_limit': 18,
1101             },
1102             'skip': 'Private video',
1103         },
1104         {
1105             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1106             'note': 'Use the first video ID in the URL',
1107             'info_dict': {
1108                 'id': 'BaW_jenozKc',
1109                 'ext': 'mp4',
1110                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1111                 'uploader': 'Philipp Hagemeister',
1112                 'uploader_id': 'phihag',
1113                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1114                 'channel': 'Philipp Hagemeister',
1115                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1116                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1117                 'upload_date': '20121002',
1118                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1119                 'categories': ['Science & Technology'],
1120                 'tags': ['youtube-dl'],
1121                 'duration': 10,
1122                 'view_count': int,
1123                 'like_count': int,
1124                 'availability': 'public',
1125                 'playable_in_embed': True,
1126                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1127                 'live_status': 'not_live',
1128                 'age_limit': 0,
1129                 'channel_follower_count': int
1130             },
1131             'params': {
1132                 'skip_download': True,
1133             },
1134         },
1135         {
1136             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1137             'note': '256k DASH audio (format 141) via DASH manifest',
1138             'info_dict': {
1139                 'id': 'a9LDPn-MO4I',
1140                 'ext': 'm4a',
1141                 'upload_date': '20121002',
1142                 'uploader_id': '8KVIDEO',
1143                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1144                 'description': '',
1145                 'uploader': '8KVIDEO',
1146                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1147             },
1148             'params': {
1149                 'youtube_include_dash_manifest': True,
1150                 'format': '141',
1151             },
1152             'skip': 'format 141 not served anymore',
1153         },
1154         # DASH manifest with encrypted signature
1155         {
1156             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1157             'info_dict': {
1158                 'id': 'IB3lcPjvWLA',
1159                 'ext': 'm4a',
1160                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1161                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1162                 'duration': 244,
1163                 'uploader': 'AfrojackVEVO',
1164                 'uploader_id': 'AfrojackVEVO',
1165                 'upload_date': '20131011',
1166                 'abr': 129.495,
1167                 'like_count': int,
1168                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1169                 'playable_in_embed': True,
1170                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1171                 'view_count': int,
1172                 'track': 'The Spark',
1173                 'live_status': 'not_live',
1174                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1175                 'channel': 'Afrojack',
1176                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1177                 'tags': 'count:19',
1178                 'availability': 'public',
1179                 'categories': ['Music'],
1180                 'age_limit': 0,
1181                 'alt_title': 'The Spark',
1182                 'channel_follower_count': int
1183             },
1184             'params': {
1185                 'youtube_include_dash_manifest': True,
1186                 'format': '141/bestaudio[ext=m4a]',
1187             },
1188         },
1189         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1190         {
1191             'note': 'Embed allowed age-gate video',
1192             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1193             'info_dict': {
1194                 'id': 'HtVdAasjOgU',
1195                 'ext': 'mp4',
1196                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1197                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1198                 'duration': 142,
1199                 'uploader': 'The Witcher',
1200                 'uploader_id': 'WitcherGame',
1201                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1202                 'upload_date': '20140605',
1203                 'age_limit': 18,
1204                 'categories': ['Gaming'],
1205                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1206                 'availability': 'needs_auth',
1207                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1208                 'like_count': int,
1209                 'channel': 'The Witcher',
1210                 'live_status': 'not_live',
1211                 'tags': 'count:17',
1212                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1213                 'playable_in_embed': True,
1214                 'view_count': int,
1215                 'channel_follower_count': int
1216             },
1217         },
1218         {
1219             'note': 'Age-gate video with embed allowed in public site',
1220             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1221             'info_dict': {
1222                 'id': 'HsUATh_Nc2U',
1223                 'ext': 'mp4',
1224                 'title': 'Godzilla 2 (Official Video)',
1225                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1226                 'upload_date': '20200408',
1227                 'uploader_id': 'FlyingKitty900',
1228                 'uploader': 'FlyingKitty',
1229                 'age_limit': 18,
1230                 'availability': 'needs_auth',
1231                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1232                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1233                 'channel': 'FlyingKitty',
1234                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1235                 'view_count': int,
1236                 'categories': ['Entertainment'],
1237                 'live_status': 'not_live',
1238                 'tags': ['Flyingkitty', 'godzilla 2'],
1239                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1240                 'like_count': int,
1241                 'duration': 177,
1242                 'playable_in_embed': True,
1243                 'channel_follower_count': int
1244             },
1245         },
1246         {
1247             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1248             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1249             'info_dict': {
1250                 'id': 'Tq92D6wQ1mg',
1251                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1252                 'ext': 'mp4',
1253                 'upload_date': '20191228',
1254                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1255                 'uploader': 'Projekt Melody',
1256                 'description': 'md5:17eccca93a786d51bc67646756894066',
1257                 'age_limit': 18,
1258                 'like_count': int,
1259                 'availability': 'needs_auth',
1260                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1261                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1262                 'view_count': int,
1263                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1264                 'channel': 'Projekt Melody',
1265                 'live_status': 'not_live',
1266                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1267                 'playable_in_embed': True,
1268                 'categories': ['Entertainment'],
1269                 'duration': 106,
1270                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1271                 'channel_follower_count': int
1272             },
1273         },
1274         {
1275             'note': 'Non-Agegated non-embeddable video',
1276             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1277             'info_dict': {
1278                 'id': 'MeJVWBSsPAY',
1279                 'ext': 'mp4',
1280                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1281                 'uploader': 'Herr Lurik',
1282                 'uploader_id': 'st3in234',
1283                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1284                 'upload_date': '20130730',
1285                 'track': 'Such mich find mich',
1286                 'age_limit': 0,
1287                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1288                 'like_count': int,
1289                 'playable_in_embed': False,
1290                 'creator': 'OOMPH!',
1291                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1292                 'view_count': int,
1293                 'alt_title': 'Such mich find mich',
1294                 'duration': 210,
1295                 'channel': 'Herr Lurik',
1296                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1297                 'categories': ['Music'],
1298                 'availability': 'public',
1299                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1300                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1301                 'live_status': 'not_live',
1302                 'artist': 'OOMPH!',
1303                 'channel_follower_count': int
1304             },
1305         },
1306         {
1307             'note': 'Non-bypassable age-gated video',
1308             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1309             'only_matching': True,
1310         },
1311         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1312         # YouTube Red ad is not captured for creator
1313         {
1314             'url': '__2ABJjxzNo',
1315             'info_dict': {
1316                 'id': '__2ABJjxzNo',
1317                 'ext': 'mp4',
1318                 'duration': 266,
1319                 'upload_date': '20100430',
1320                 'uploader_id': 'deadmau5',
1321                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1322                 'creator': 'deadmau5',
1323                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1324                 'uploader': 'deadmau5',
1325                 'title': 'Deadmau5 - Some Chords (HD)',
1326                 'alt_title': 'Some Chords',
1327                 'availability': 'public',
1328                 'tags': 'count:14',
1329                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1330                 'view_count': int,
1331                 'live_status': 'not_live',
1332                 'channel': 'deadmau5',
1333                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1334                 'like_count': int,
1335                 'track': 'Some Chords',
1336                 'artist': 'deadmau5',
1337                 'playable_in_embed': True,
1338                 'age_limit': 0,
1339                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1340                 'categories': ['Music'],
1341                 'album': 'Some Chords',
1342                 'channel_follower_count': int
1343             },
1344             'expected_warnings': [
1345                 'DASH manifest missing',
1346             ]
1347         },
1348         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1349         {
1350             'url': 'lqQg6PlCWgI',
1351             'info_dict': {
1352                 'id': 'lqQg6PlCWgI',
1353                 'ext': 'mp4',
1354                 'duration': 6085,
1355                 'upload_date': '20150827',
1356                 'uploader_id': 'olympic',
1357                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1358                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1359                 'uploader': 'Olympics',
1360                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1361                 'like_count': int,
1362                 'release_timestamp': 1343767800,
1363                 'playable_in_embed': True,
1364                 'categories': ['Sports'],
1365                 'release_date': '20120731',
1366                 'channel': 'Olympics',
1367                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1368                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1369                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1370                 'age_limit': 0,
1371                 'availability': 'public',
1372                 'live_status': 'was_live',
1373                 'view_count': int,
1374                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1375                 'channel_follower_count': int
1376             },
1377             'params': {
1378                 'skip_download': 'requires avconv',
1379             }
1380         },
1381         # Non-square pixels
1382         {
1383             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1384             'info_dict': {
1385                 'id': '_b-2C3KPAM0',
1386                 'ext': 'mp4',
1387                 'stretched_ratio': 16 / 9.,
1388                 'duration': 85,
1389                 'upload_date': '20110310',
1390                 'uploader_id': 'AllenMeow',
1391                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1392                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1393                 'uploader': '孫ᄋᄅ',
1394                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1395                 'playable_in_embed': True,
1396                 'channel': '孫ᄋᄅ',
1397                 'age_limit': 0,
1398                 'tags': 'count:11',
1399                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1400                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1401                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1402                 'view_count': int,
1403                 'categories': ['People & Blogs'],
1404                 'like_count': int,
1405                 'live_status': 'not_live',
1406                 'availability': 'unlisted',
1407                 'channel_follower_count': int
1408             },
1409         },
1410         # url_encoded_fmt_stream_map is empty string
1411         {
1412             'url': 'qEJwOuvDf7I',
1413             'info_dict': {
1414                 'id': 'qEJwOuvDf7I',
1415                 'ext': 'webm',
1416                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1417                 'description': '',
1418                 'upload_date': '20150404',
1419                 'uploader_id': 'spbelect',
1420                 'uploader': 'Наблюдатели Петербурга',
1421             },
1422             'params': {
1423                 'skip_download': 'requires avconv',
1424             },
1425             'skip': 'This live event has ended.',
1426         },
1427         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1428         {
1429             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1430             'info_dict': {
1431                 'id': 'FIl7x6_3R5Y',
1432                 'ext': 'webm',
1433                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1434                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1435                 'duration': 220,
1436                 'upload_date': '20150625',
1437                 'uploader_id': 'dorappi2000',
1438                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1439                 'uploader': 'dorappi2000',
1440                 'formats': 'mincount:31',
1441             },
1442             'skip': 'not actual anymore',
1443         },
1444         # DASH manifest with segment_list
1445         {
1446             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1447             'md5': '8ce563a1d667b599d21064e982ab9e31',
1448             'info_dict': {
1449                 'id': 'CsmdDsKjzN8',
1450                 'ext': 'mp4',
1451                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1452                 'uploader': 'Airtek',
1453                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1454                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1455                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1456             },
1457             'params': {
1458                 'youtube_include_dash_manifest': True,
1459                 'format': '135',  # bestvideo
1460             },
1461             'skip': 'This live event has ended.',
1462         },
1463         {
1464             # Multifeed videos (multiple cameras), URL is for Main Camera
1465             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1466             'info_dict': {
1467                 'id': 'jvGDaLqkpTg',
1468                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1469                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1470             },
1471             'playlist': [{
1472                 'info_dict': {
1473                     'id': 'jvGDaLqkpTg',
1474                     'ext': 'mp4',
1475                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1476                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1477                     'duration': 10643,
1478                     'upload_date': '20161111',
1479                     'uploader': 'Team PGP',
1480                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1481                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1482                 },
1483             }, {
1484                 'info_dict': {
1485                     'id': '3AKt1R1aDnw',
1486                     'ext': 'mp4',
1487                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1488                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1489                     'duration': 10991,
1490                     'upload_date': '20161111',
1491                     'uploader': 'Team PGP',
1492                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1493                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1494                 },
1495             }, {
1496                 'info_dict': {
1497                     'id': 'RtAMM00gpVc',
1498                     'ext': 'mp4',
1499                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1500                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1501                     'duration': 10995,
1502                     'upload_date': '20161111',
1503                     'uploader': 'Team PGP',
1504                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1505                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1506                 },
1507             }, {
1508                 'info_dict': {
1509                     'id': '6N2fdlP3C5U',
1510                     'ext': 'mp4',
1511                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1512                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1513                     'duration': 10990,
1514                     'upload_date': '20161111',
1515                     'uploader': 'Team PGP',
1516                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1517                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1518                 },
1519             }],
1520             'params': {
1521                 'skip_download': True,
1522             },
1523             'skip': 'Not multifeed anymore',
1524         },
1525         {
1526             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1527             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1528             'info_dict': {
1529                 'id': 'gVfLd0zydlo',
1530                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1531             },
1532             'playlist_count': 2,
1533             'skip': 'Not multifeed anymore',
1534         },
1535         {
1536             'url': 'https://vid.plus/FlRa-iH7PGw',
1537             'only_matching': True,
1538         },
1539         {
1540             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1541             'only_matching': True,
1542         },
1543         {
1544             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1545             # Also tests cut-off URL expansion in video description (see
1546             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1547             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1548             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1549             'info_dict': {
1550                 'id': 'lsguqyKfVQg',
1551                 'ext': 'mp4',
1552                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1553                 'alt_title': 'Dark Walk',
1554                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1555                 'duration': 133,
1556                 'upload_date': '20151119',
1557                 'uploader_id': 'IronSoulElf',
1558                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1559                 'uploader': 'IronSoulElf',
1560                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1561                 'track': 'Dark Walk',
1562                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1563                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1564                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1565                 'categories': ['Film & Animation'],
1566                 'view_count': int,
1567                 'live_status': 'not_live',
1568                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1569                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1570                 'tags': 'count:13',
1571                 'availability': 'public',
1572                 'channel': 'IronSoulElf',
1573                 'playable_in_embed': True,
1574                 'like_count': int,
1575                 'age_limit': 0,
1576                 'channel_follower_count': int
1577             },
1578             'params': {
1579                 'skip_download': True,
1580             },
1581         },
1582         {
1583             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1584             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1585             'only_matching': True,
1586         },
1587         {
1588             # Video with yt:stretch=17:0
1589             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1590             'info_dict': {
1591                 'id': 'Q39EVAstoRM',
1592                 'ext': 'mp4',
1593                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1594                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1595                 'upload_date': '20151107',
1596                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1597                 'uploader': 'CH GAMER DROID',
1598             },
1599             'params': {
1600                 'skip_download': True,
1601             },
1602             'skip': 'This video does not exist.',
1603         },
1604         {
1605             # Video with incomplete 'yt:stretch=16:'
1606             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1607             'only_matching': True,
1608         },
1609         {
1610             # Video licensed under Creative Commons
1611             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1612             'info_dict': {
1613                 'id': 'M4gD1WSo5mA',
1614                 'ext': 'mp4',
1615                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1616                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1617                 'duration': 721,
1618                 'upload_date': '20150128',
1619                 'uploader_id': 'BerkmanCenter',
1620                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1621                 'uploader': 'The Berkman Klein Center for Internet & Society',
1622                 'license': 'Creative Commons Attribution license (reuse allowed)',
1623                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1624                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1625                 'like_count': int,
1626                 'age_limit': 0,
1627                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1628                 'channel': 'The Berkman Klein Center for Internet & Society',
1629                 'availability': 'public',
1630                 'view_count': int,
1631                 'categories': ['Education'],
1632                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1633                 'live_status': 'not_live',
1634                 'playable_in_embed': True,
1635                 'channel_follower_count': int
1636             },
1637             'params': {
1638                 'skip_download': True,
1639             },
1640         },
1641         {
1642             # Channel-like uploader_url
1643             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1644             'info_dict': {
1645                 'id': 'eQcmzGIKrzg',
1646                 'ext': 'mp4',
1647                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1648                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1649                 'duration': 4060,
1650                 'upload_date': '20151120',
1651                 'uploader': 'Bernie Sanders',
1652                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1653                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1654                 'license': 'Creative Commons Attribution license (reuse allowed)',
1655                 'playable_in_embed': True,
1656                 'tags': 'count:12',
1657                 'like_count': int,
1658                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1659                 'age_limit': 0,
1660                 'availability': 'public',
1661                 'categories': ['News & Politics'],
1662                 'channel': 'Bernie Sanders',
1663                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1664                 'view_count': int,
1665                 'live_status': 'not_live',
1666                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1667                 'channel_follower_count': int
1668             },
1669             'params': {
1670                 'skip_download': True,
1671             },
1672         },
1673         {
1674             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1675             'only_matching': True,
1676         },
1677         {
1678             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1679             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1680             'only_matching': True,
1681         },
1682         {
1683             # Rental video preview
1684             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1685             'info_dict': {
1686                 'id': 'uGpuVWrhIzE',
1687                 'ext': 'mp4',
1688                 'title': 'Piku - Trailer',
1689                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1690                 'upload_date': '20150811',
1691                 'uploader': 'FlixMatrix',
1692                 'uploader_id': 'FlixMatrixKaravan',
1693                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1694                 'license': 'Standard YouTube License',
1695             },
1696             'params': {
1697                 'skip_download': True,
1698             },
1699             'skip': 'This video is not available.',
1700         },
1701         {
1702             # YouTube Red video with episode data
1703             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1704             'info_dict': {
1705                 'id': 'iqKdEhx-dD4',
1706                 'ext': 'mp4',
1707                 'title': 'Isolation - Mind Field (Ep 1)',
1708                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1709                 'duration': 2085,
1710                 'upload_date': '20170118',
1711                 'uploader': 'Vsauce',
1712                 'uploader_id': 'Vsauce',
1713                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1714                 'series': 'Mind Field',
1715                 'season_number': 1,
1716                 'episode_number': 1,
1717                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1718                 'tags': 'count:12',
1719                 'view_count': int,
1720                 'availability': 'public',
1721                 'age_limit': 0,
1722                 'channel': 'Vsauce',
1723                 'episode': 'Episode 1',
1724                 'categories': ['Entertainment'],
1725                 'season': 'Season 1',
1726                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1727                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1728                 'like_count': int,
1729                 'playable_in_embed': True,
1730                 'live_status': 'not_live',
1731                 'channel_follower_count': int
1732             },
1733             'params': {
1734                 'skip_download': True,
1735             },
1736             'expected_warnings': [
1737                 'Skipping DASH manifest',
1738             ],
1739         },
1740         {
1741             # The following content has been identified by the YouTube community
1742             # as inappropriate or offensive to some audiences.
1743             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1744             'info_dict': {
1745                 'id': '6SJNVb0GnPI',
1746                 'ext': 'mp4',
1747                 'title': 'Race Differences in Intelligence',
1748                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1749                 'duration': 965,
1750                 'upload_date': '20140124',
1751                 'uploader': 'New Century Foundation',
1752                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1753                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1754             },
1755             'params': {
1756                 'skip_download': True,
1757             },
1758             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1759         },
1760         {
1761             # itag 212
1762             'url': '1t24XAntNCY',
1763             'only_matching': True,
1764         },
1765         {
1766             # geo restricted to JP
1767             'url': 'sJL6WA-aGkQ',
1768             'only_matching': True,
1769         },
1770         {
1771             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1772             'only_matching': True,
1773         },
1774         {
1775             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1776             'only_matching': True,
1777         },
1778         {
1779             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1780             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1781             'only_matching': True,
1782         },
1783         {
1784             # DRM protected
1785             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1786             'only_matching': True,
1787         },
1788         {
1789             # Video with unsupported adaptive stream type formats
1790             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1791             'info_dict': {
1792                 'id': 'Z4Vy8R84T1U',
1793                 'ext': 'mp4',
1794                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1795                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1796                 'duration': 433,
1797                 'upload_date': '20130923',
1798                 'uploader': 'Amelia Putri Harwita',
1799                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1800                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1801                 'formats': 'maxcount:10',
1802             },
1803             'params': {
1804                 'skip_download': True,
1805                 'youtube_include_dash_manifest': False,
1806             },
1807             'skip': 'not actual anymore',
1808         },
1809         {
1810             # Youtube Music Auto-generated description
1811             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1812             'info_dict': {
1813                 'id': 'MgNrAu2pzNs',
1814                 'ext': 'mp4',
1815                 'title': 'Voyeur Girl',
1816                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1817                 'upload_date': '20190312',
1818                 'uploader': 'Stephen - Topic',
1819                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1820                 'artist': 'Stephen',
1821                 'track': 'Voyeur Girl',
1822                 'album': 'it\'s too much love to know my dear',
1823                 'release_date': '20190313',
1824                 'release_year': 2019,
1825                 'alt_title': 'Voyeur Girl',
1826                 'view_count': int,
1827                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1828                 'playable_in_embed': True,
1829                 'like_count': int,
1830                 'categories': ['Music'],
1831                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1832                 'channel': 'Stephen',
1833                 'availability': 'public',
1834                 'creator': 'Stephen',
1835                 'duration': 169,
1836                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1837                 'age_limit': 0,
1838                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1839                 'tags': 'count:11',
1840                 'live_status': 'not_live',
1841                 'channel_follower_count': int
1842             },
1843             'params': {
1844                 'skip_download': True,
1845             },
1846         },
1847         {
1848             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1849             'only_matching': True,
1850         },
1851         {
1852             # invalid -> valid video id redirection
1853             'url': 'DJztXj2GPfl',
1854             'info_dict': {
1855                 'id': 'DJztXj2GPfk',
1856                 'ext': 'mp4',
1857                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1858                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1859                 'upload_date': '20090125',
1860                 'uploader': 'Prochorowka',
1861                 'uploader_id': 'Prochorowka',
1862                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1863                 'artist': 'Panjabi MC',
1864                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1865                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1866             },
1867             'params': {
1868                 'skip_download': True,
1869             },
1870             'skip': 'Video unavailable',
1871         },
1872         {
1873             # empty description results in an empty string
1874             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1875             'info_dict': {
1876                 'id': 'x41yOUIvK2k',
1877                 'ext': 'mp4',
1878                 'title': 'IMG 3456',
1879                 'description': '',
1880                 'upload_date': '20170613',
1881                 'uploader_id': 'ElevageOrVert',
1882                 'uploader': 'ElevageOrVert',
1883                 'view_count': int,
1884                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1885                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1886                 'like_count': int,
1887                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1888                 'tags': [],
1889                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1890                 'availability': 'public',
1891                 'age_limit': 0,
1892                 'categories': ['Pets & Animals'],
1893                 'duration': 7,
1894                 'playable_in_embed': True,
1895                 'live_status': 'not_live',
1896                 'channel': 'ElevageOrVert',
1897                 'channel_follower_count': int
1898             },
1899             'params': {
1900                 'skip_download': True,
1901             },
1902         },
1903         {
1904             # with '};' inside yt initial data (see [1])
1905             # see [2] for an example with '};' inside ytInitialPlayerResponse
1906             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1907             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1908             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1909             'info_dict': {
1910                 'id': 'CHqg6qOn4no',
1911                 'ext': 'mp4',
1912                 'title': 'Part 77   Sort a list of simple types in c#',
1913                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1914                 'upload_date': '20130831',
1915                 'uploader_id': 'kudvenkat',
1916                 'uploader': 'kudvenkat',
1917                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1918                 'like_count': int,
1919                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1920                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1921                 'live_status': 'not_live',
1922                 'categories': ['Education'],
1923                 'availability': 'public',
1924                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1925                 'tags': 'count:12',
1926                 'playable_in_embed': True,
1927                 'age_limit': 0,
1928                 'view_count': int,
1929                 'duration': 522,
1930                 'channel': 'kudvenkat',
1931                 'channel_follower_count': int
1932             },
1933             'params': {
1934                 'skip_download': True,
1935             },
1936         },
1937         {
1938             # another example of '};' in ytInitialData
1939             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1940             'only_matching': True,
1941         },
1942         {
1943             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1944             'only_matching': True,
1945         },
1946         {
1947             # https://github.com/ytdl-org/youtube-dl/pull/28094
1948             'url': 'OtqTfy26tG0',
1949             'info_dict': {
1950                 'id': 'OtqTfy26tG0',
1951                 'ext': 'mp4',
1952                 'title': 'Burn Out',
1953                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1954                 'upload_date': '20141120',
1955                 'uploader': 'The Cinematic Orchestra - Topic',
1956                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1957                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1958                 'artist': 'The Cinematic Orchestra',
1959                 'track': 'Burn Out',
1960                 'album': 'Every Day',
1961                 'like_count': int,
1962                 'live_status': 'not_live',
1963                 'alt_title': 'Burn Out',
1964                 'duration': 614,
1965                 'age_limit': 0,
1966                 'view_count': int,
1967                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1968                 'creator': 'The Cinematic Orchestra',
1969                 'channel': 'The Cinematic Orchestra',
1970                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1971                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1972                 'availability': 'public',
1973                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1974                 'categories': ['Music'],
1975                 'playable_in_embed': True,
1976                 'channel_follower_count': int
1977             },
1978             'params': {
1979                 'skip_download': True,
1980             },
1981         },
1982         {
1983             # controversial video, only works with bpctr when authenticated with cookies
1984             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1985             'only_matching': True,
1986         },
1987         {
1988             # controversial video, requires bpctr/contentCheckOk
1989             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1990             'info_dict': {
1991                 'id': 'SZJvDhaSDnc',
1992                 'ext': 'mp4',
1993                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1994                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1995                 'uploader': 'CBS Mornings',
1996                 'uploader_id': 'CBSThisMorning',
1997                 'upload_date': '20140716',
1998                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1999                 'duration': 170,
2000                 'categories': ['News & Politics'],
2001                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2002                 'view_count': int,
2003                 'channel': 'CBS Mornings',
2004                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2005                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2006                 'age_limit': 18,
2007                 'availability': 'needs_auth',
2008                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2009                 'like_count': int,
2010                 'live_status': 'not_live',
2011                 'playable_in_embed': True,
2012                 'channel_follower_count': int
2013             }
2014         },
2015         {
2016             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2017             'url': 'cBvYw8_A0vQ',
2018             'info_dict': {
2019                 'id': 'cBvYw8_A0vQ',
2020                 'ext': 'mp4',
2021                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2022                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2023                 'upload_date': '20201120',
2024                 'uploader': 'Walk around Japan',
2025                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2026                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2027                 'duration': 1456,
2028                 'categories': ['Travel & Events'],
2029                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2030                 'view_count': int,
2031                 'channel': 'Walk around Japan',
2032                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2033                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2034                 'age_limit': 0,
2035                 'availability': 'public',
2036                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2037                 'live_status': 'not_live',
2038                 'playable_in_embed': True,
2039                 'channel_follower_count': int
2040             },
2041             'params': {
2042                 'skip_download': True,
2043             },
2044         }, {
2045             # Has multiple audio streams
2046             'url': 'WaOKSUlf4TM',
2047             'only_matching': True
2048         }, {
2049             # Requires Premium: has format 141 when requested using YTM url
2050             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2051             'only_matching': True
2052         }, {
2053             # multiple subtitles with same lang_code
2054             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2055             'only_matching': True,
2056         }, {
2057             # Force use android client fallback
2058             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2059             'info_dict': {
2060                 'id': 'YOelRv7fMxY',
2061                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2062                 'ext': '3gp',
2063                 'upload_date': '20210624',
2064                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2065                 'uploader': 'colinfurze',
2066                 'uploader_id': 'colinfurze',
2067                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2068                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2069                 'duration': 596,
2070                 'categories': ['Entertainment'],
2071                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2072                 'view_count': int,
2073                 'channel': 'colinfurze',
2074                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2075                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2076                 'age_limit': 0,
2077                 'availability': 'public',
2078                 'like_count': int,
2079                 'live_status': 'not_live',
2080                 'playable_in_embed': True,
2081                 'channel_follower_count': int
2082             },
2083             'params': {
2084                 'format': '17',  # 3gp format available on android
2085                 'extractor_args': {'youtube': {'player_client': ['android']}},
2086             },
2087         },
2088         {
2089             # Skip download of additional client configs (remix client config in this case)
2090             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2091             'only_matching': True,
2092             'params': {
2093                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2094             },
2095         }, {
2096             # shorts
2097             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2098             'only_matching': True,
2099         }, {
2100             'note': 'Storyboards',
2101             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2102             'info_dict': {
2103                 'id': '5KLPxDtMqe8',
2104                 'ext': 'mhtml',
2105                 'format_id': 'sb0',
2106                 'title': 'Your Brain is Plastic',
2107                 'uploader_id': 'scishow',
2108                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2109                 'upload_date': '20140324',
2110                 'uploader': 'SciShow',
2111                 'like_count': int,
2112                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2113                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2114                 'view_count': int,
2115                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2116                 'playable_in_embed': True,
2117                 'tags': 'count:12',
2118                 'uploader_url': 'http://www.youtube.com/user/scishow',
2119                 'availability': 'public',
2120                 'channel': 'SciShow',
2121                 'live_status': 'not_live',
2122                 'duration': 248,
2123                 'categories': ['Education'],
2124                 'age_limit': 0,
2125                 'channel_follower_count': int
2126             }, 'params': {'format': 'mhtml', 'skip_download': True}
2127         }, {
2128             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2129             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2130             'info_dict': {
2131                 'id': '2NUZ8W2llS4',
2132                 'ext': 'mp4',
2133                 'title': 'The NP that test your phone performance 🙂',
2134                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2135                 'uploader': 'Leon Nguyen',
2136                 'uploader_id': 'VNSXIII',
2137                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2138                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2139                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2140                 'duration': 21,
2141                 'view_count': int,
2142                 'age_limit': 0,
2143                 'categories': ['Gaming'],
2144                 'tags': 'count:23',
2145                 'playable_in_embed': True,
2146                 'live_status': 'not_live',
2147                 'upload_date': '20220103',
2148                 'like_count': int,
2149                 'availability': 'public',
2150                 'channel': 'Leon Nguyen',
2151                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2152                 'channel_follower_count': int
2153             }
2154         }, {
2155             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2156             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2157             'info_dict': {
2158                 'id': 'mzZzzBU6lrM',
2159                 'ext': 'mp4',
2160                 'title': 'I Met GeorgeNotFound In Real Life...',
2161                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2162                 'uploader': 'Quackity',
2163                 'uploader_id': 'QuackityHQ',
2164                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2165                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2166                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2167                 'duration': 955,
2168                 'view_count': int,
2169                 'age_limit': 0,
2170                 'categories': ['Entertainment'],
2171                 'tags': 'count:26',
2172                 'playable_in_embed': True,
2173                 'live_status': 'not_live',
2174                 'release_timestamp': 1641172509,
2175                 'release_date': '20220103',
2176                 'upload_date': '20220103',
2177                 'like_count': int,
2178                 'availability': 'public',
2179                 'channel': 'Quackity',
2180                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2181                 'channel_follower_count': int
2182             }
2183         },
2184         {   # continuous livestream. Microformat upload date should be preferred.
2185             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2186             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2187             'info_dict': {
2188                 'id': 'kgx4WGK0oNU',
2189                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2190                 'ext': 'mp4',
2191                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2192                 'availability': 'public',
2193                 'age_limit': 0,
2194                 'release_timestamp': 1637975704,
2195                 'upload_date': '20210619',
2196                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2197                 'live_status': 'is_live',
2198                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2199                 'uploader': '阿鲍Abao',
2200                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2201                 'channel': 'Abao in Tokyo',
2202                 'channel_follower_count': int,
2203                 'release_date': '20211127',
2204                 'tags': 'count:39',
2205                 'categories': ['People & Blogs'],
2206                 'like_count': int,
2207                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2208                 'view_count': int,
2209                 'playable_in_embed': True,
2210                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2211             },
2212             'params': {'skip_download': True}
2213         }, {
2214             # Story. Requires specific player params to work.
2215             # Note: stories get removed after some period of time
2216             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2217             'info_dict': {
2218                 'id': 'vv8qTUWmulI',
2219                 'ext': 'mp4',
2220                 'availability': 'unlisted',
2221                 'view_count': int,
2222                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2223                 'upload_date': '20220526',
2224                 'categories': ['Education'],
2225                 'title': 'Story',
2226                 'channel': 'IT\'S HISTORY',
2227                 'description': '',
2228                 'uploader_id': 'BlastfromthePast',
2229                 'duration': 12,
2230                 'uploader': 'IT\'S HISTORY',
2231                 'playable_in_embed': True,
2232                 'age_limit': 0,
2233                 'live_status': 'not_live',
2234                 'tags': [],
2235                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2236                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2237                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2238             }
2239         }, {
2240             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2241             'info_dict': {
2242                 'id': 'tjjjtzRLHvA',
2243                 'ext': 'mp4',
2244                 'title': 'ハッシュタグ無し };if window.ytcsi',
2245                 'upload_date': '20220323',
2246                 'like_count': int,
2247                 'availability': 'unlisted',
2248                 'channel': 'nao20010128nao',
2249                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2250                 'age_limit': 0,
2251                 'uploader': 'nao20010128nao',
2252                 'uploader_id': 'nao20010128nao',
2253                 'categories': ['Music'],
2254                 'view_count': int,
2255                 'description': '',
2256                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2257                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2258                 'live_status': 'not_live',
2259                 'playable_in_embed': True,
2260                 'channel_follower_count': int,
2261                 'duration': 6,
2262                 'tags': [],
2263                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2264             }
2265         }
2266     ]
2267
2268     @classmethod
2269     def suitable(cls, url):
2270         from ..utils import parse_qs
2271
2272         qs = parse_qs(url)
2273         if qs.get('list', [None])[0]:
2274             return False
2275         return super().suitable(url)
2276
2277     def __init__(self, *args, **kwargs):
2278         super().__init__(*args, **kwargs)
2279         self._code_cache = {}
2280         self._player_cache = {}
2281
2282     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2283         lock = threading.Lock()
2284
2285         is_live = True
2286         start_time = time.time()
2287         formats = [f for f in formats if f.get('is_from_start')]
2288
2289         def refetch_manifest(format_id, delay):
2290             nonlocal formats, start_time, is_live
2291             if time.time() <= start_time + delay:
2292                 return
2293
2294             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2295             video_details = traverse_obj(
2296                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2297             microformats = traverse_obj(
2298                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2299                 expected_type=dict, default=[])
2300             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2301             start_time = time.time()
2302
2303         def mpd_feed(format_id, delay):
2304             """
2305             @returns (manifest_url, manifest_stream_number, is_live) or None
2306             """
2307             with lock:
2308                 refetch_manifest(format_id, delay)
2309
2310             f = next((f for f in formats if f['format_id'] == format_id), None)
2311             if not f:
2312                 if not is_live:
2313                     self.to_screen(f'{video_id}: Video is no longer live')
2314                 else:
2315                     self.report_warning(
2316                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2317                 return None
2318             return f['manifest_url'], f['manifest_stream_number'], is_live
2319
2320         for f in formats:
2321             f['is_live'] = True
2322             f['protocol'] = 'http_dash_segments_generator'
2323             f['fragments'] = functools.partial(
2324                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2325
2326     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2327         FETCH_SPAN, MAX_DURATION = 5, 432000
2328
2329         mpd_url, stream_number, is_live = None, None, True
2330
2331         begin_index = 0
2332         download_start_time = ctx.get('start') or time.time()
2333
2334         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2335         if lack_early_segments:
2336             self.report_warning(bug_reports_message(
2337                 'Starting download from the last 120 hours of the live stream since '
2338                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2339             lack_early_segments = True
2340
2341         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2342         fragments, fragment_base_url = None, None
2343
2344         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2345             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2346             # Obtain from MPD's maximum seq value
2347             old_mpd_url = mpd_url
2348             last_error = ctx.pop('last_error', None)
2349             expire_fast = immediate or last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2350             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2351                                                or (mpd_url, stream_number, False))
2352             if not refresh_sequence:
2353                 if expire_fast and not is_live:
2354                     return False, last_seq
2355                 elif old_mpd_url == mpd_url:
2356                     return True, last_seq
2357             try:
2358                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2359                     mpd_url, None, note=False, errnote=False, fatal=False)
2360             except ExtractorError:
2361                 fmts = None
2362             if not fmts:
2363                 no_fragment_score += 2
2364                 return False, last_seq
2365             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2366             fragments = fmt_info['fragments']
2367             fragment_base_url = fmt_info['fragment_base_url']
2368             assert fragment_base_url
2369
2370             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2371             return True, _last_seq
2372
2373         while is_live:
2374             fetch_time = time.time()
2375             if no_fragment_score > 30:
2376                 return
2377             if last_segment_url:
2378                 # Obtain from "X-Head-Seqnum" header value from each segment
2379                 try:
2380                     urlh = self._request_webpage(
2381                         last_segment_url, None, note=False, errnote=False, fatal=False)
2382                 except ExtractorError:
2383                     urlh = None
2384                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2385                 if last_seq is None:
2386                     no_fragment_score += 2
2387                     last_segment_url = None
2388                     continue
2389             else:
2390                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2391                 no_fragment_score += 2
2392                 if not should_continue:
2393                     continue
2394
2395             if known_idx > last_seq:
2396                 last_segment_url = None
2397                 continue
2398
2399             last_seq += 1
2400
2401             if begin_index < 0 and known_idx < 0:
2402                 # skip from the start when it's negative value
2403                 known_idx = last_seq + begin_index
2404             if lack_early_segments:
2405                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2406             try:
2407                 for idx in range(known_idx, last_seq):
2408                     # do not update sequence here or you'll get skipped some part of it
2409                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2410                     if not should_continue:
2411                         known_idx = idx - 1
2412                         raise ExtractorError('breaking out of outer loop')
2413                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2414                     yield {
2415                         'url': last_segment_url,
2416                         'fragment_count': last_seq,
2417                     }
2418                 if known_idx == last_seq:
2419                     no_fragment_score += 5
2420                 else:
2421                     no_fragment_score = 0
2422                 known_idx = last_seq
2423             except ExtractorError:
2424                 continue
2425
2426             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2427
2428     def _extract_player_url(self, *ytcfgs, webpage=None):
2429         player_url = traverse_obj(
2430             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2431             get_all=False, expected_type=compat_str)
2432         if not player_url:
2433             return
2434         return urljoin('https://www.youtube.com', player_url)
2435
2436     def _download_player_url(self, video_id, fatal=False):
2437         res = self._download_webpage(
2438             'https://www.youtube.com/iframe_api',
2439             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2440         if res:
2441             player_version = self._search_regex(
2442                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2443             if player_version:
2444                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2445
2446     def _signature_cache_id(self, example_sig):
2447         """ Return a string representation of a signature """
2448         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2449
2450     @classmethod
2451     def _extract_player_info(cls, player_url):
2452         for player_re in cls._PLAYER_INFO_RE:
2453             id_m = re.search(player_re, player_url)
2454             if id_m:
2455                 break
2456         else:
2457             raise ExtractorError('Cannot identify player %r' % player_url)
2458         return id_m.group('id')
2459
2460     def _load_player(self, video_id, player_url, fatal=True):
2461         player_id = self._extract_player_info(player_url)
2462         if player_id not in self._code_cache:
2463             code = self._download_webpage(
2464                 player_url, video_id, fatal=fatal,
2465                 note='Downloading player ' + player_id,
2466                 errnote='Download of %s failed' % player_url)
2467             if code:
2468                 self._code_cache[player_id] = code
2469         return self._code_cache.get(player_id)
2470
2471     def _extract_signature_function(self, video_id, player_url, example_sig):
2472         player_id = self._extract_player_info(player_url)
2473
2474         # Read from filesystem cache
2475         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2476         assert os.path.basename(func_id) == func_id
2477
2478         cache_spec = self.cache.load('youtube-sigfuncs', func_id)
2479         if cache_spec is not None:
2480             return lambda s: ''.join(s[i] for i in cache_spec)
2481
2482         code = self._load_player(video_id, player_url)
2483         if code:
2484             res = self._parse_sig_js(code)
2485
2486             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2487             cache_res = res(test_string)
2488             cache_spec = [ord(c) for c in cache_res]
2489
2490             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2491             return res
2492
2493     def _print_sig_code(self, func, example_sig):
2494         if not self.get_param('youtube_print_sig_code'):
2495             return
2496
2497         def gen_sig_code(idxs):
2498             def _genslice(start, end, step):
2499                 starts = '' if start == 0 else str(start)
2500                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2501                 steps = '' if step == 1 else (':%d' % step)
2502                 return f's[{starts}{ends}{steps}]'
2503
2504             step = None
2505             # Quelch pyflakes warnings - start will be set when step is set
2506             start = '(Never used)'
2507             for i, prev in zip(idxs[1:], idxs[:-1]):
2508                 if step is not None:
2509                     if i - prev == step:
2510                         continue
2511                     yield _genslice(start, prev, step)
2512                     step = None
2513                     continue
2514                 if i - prev in [-1, 1]:
2515                     step = i - prev
2516                     start = prev
2517                     continue
2518                 else:
2519                     yield 's[%d]' % prev
2520             if step is None:
2521                 yield 's[%d]' % i
2522             else:
2523                 yield _genslice(start, i, step)
2524
2525         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2526         cache_res = func(test_string)
2527         cache_spec = [ord(c) for c in cache_res]
2528         expr_code = ' + '.join(gen_sig_code(cache_spec))
2529         signature_id_tuple = '(%s)' % (
2530             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2531         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2532                 '    return %s\n') % (signature_id_tuple, expr_code)
2533         self.to_screen('Extracted signature function:\n' + code)
2534
2535     def _parse_sig_js(self, jscode):
2536         funcname = self._search_regex(
2537             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2538              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2539              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2540              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2541              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2542              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2543              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2544              # Obsolete patterns
2545              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2546              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2547              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2548              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2549              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2550              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2551              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2552              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2553             jscode, 'Initial JS player signature function name', group='sig')
2554
2555         jsi = JSInterpreter(jscode)
2556         initial_function = jsi.extract_function(funcname)
2557         return lambda s: initial_function([s])
2558
2559     def _decrypt_signature(self, s, video_id, player_url):
2560         """Turn the encrypted s field into a working signature"""
2561         try:
2562             player_id = (player_url, self._signature_cache_id(s))
2563             if player_id not in self._player_cache:
2564                 func = self._extract_signature_function(video_id, player_url, s)
2565                 self._player_cache[player_id] = func
2566             func = self._player_cache[player_id]
2567             self._print_sig_code(func, s)
2568             return func(s)
2569         except Exception as e:
2570             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2571
2572     def _decrypt_nsig(self, s, video_id, player_url):
2573         """Turn the encrypted n field into a working signature"""
2574         if player_url is None:
2575             raise ExtractorError('Cannot decrypt nsig without player_url')
2576         player_url = urljoin('https://www.youtube.com', player_url)
2577
2578         sig_id = ('nsig_value', s)
2579         if sig_id in self._player_cache:
2580             return self._player_cache[sig_id]
2581
2582         try:
2583             player_id = ('nsig', player_url)
2584             if player_id not in self._player_cache:
2585                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2586             func = self._player_cache[player_id]
2587             self._player_cache[sig_id] = func(s)
2588             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2589             return self._player_cache[sig_id]
2590         except Exception as e:
2591             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2592
2593     def _extract_n_function_name(self, jscode):
2594         nfunc, idx = self._search_regex(
2595             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2596             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2597         if not idx:
2598             return nfunc
2599         return json.loads(js_to_json(self._search_regex(
2600             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2601             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2602
2603     def _extract_n_function(self, video_id, player_url):
2604         player_id = self._extract_player_info(player_url)
2605         func_code = self.cache.load('youtube-nsig', player_id)
2606
2607         if func_code:
2608             jsi = JSInterpreter(func_code)
2609         else:
2610             jscode = self._load_player(video_id, player_url)
2611             funcname = self._extract_n_function_name(jscode)
2612             jsi = JSInterpreter(jscode)
2613             func_code = jsi.extract_function_code(funcname)
2614             self.cache.store('youtube-nsig', player_id, func_code)
2615
2616         if self.get_param('youtube_print_sig_code'):
2617             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2618
2619         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2620
2621     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2622         """
2623         Extract signatureTimestamp (sts)
2624         Required to tell API what sig/player version is in use.
2625         """
2626         sts = None
2627         if isinstance(ytcfg, dict):
2628             sts = int_or_none(ytcfg.get('STS'))
2629
2630         if not sts:
2631             # Attempt to extract from player
2632             if player_url is None:
2633                 error_msg = 'Cannot extract signature timestamp without player_url.'
2634                 if fatal:
2635                     raise ExtractorError(error_msg)
2636                 self.report_warning(error_msg)
2637                 return
2638             code = self._load_player(video_id, player_url, fatal=fatal)
2639             if code:
2640                 sts = int_or_none(self._search_regex(
2641                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2642                     'JS player signature timestamp', group='sts', fatal=fatal))
2643         return sts
2644
2645     def _mark_watched(self, video_id, player_responses):
2646         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2647             label = 'fully ' if is_full else ''
2648             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2649                             expected_type=url_or_none)
2650             if not url:
2651                 self.report_warning(f'Unable to mark {label}watched')
2652                 return
2653             parsed_url = compat_urlparse.urlparse(url)
2654             qs = compat_urlparse.parse_qs(parsed_url.query)
2655
2656             # cpn generation algorithm is reverse engineered from base.js.
2657             # In fact it works even with dummy cpn.
2658             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2659             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2660
2661             # # more consistent results setting it to right before the end
2662             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2663
2664             qs.update({
2665                 'ver': ['2'],
2666                 'cpn': [cpn],
2667                 'cmt': video_length,
2668                 'el': 'detailpage',  # otherwise defaults to "shorts"
2669             })
2670
2671             if is_full:
2672                 # these seem to mark watchtime "history" in the real world
2673                 # they're required, so send in a single value
2674                 qs.update({
2675                     'st': video_length,
2676                     'et': video_length,
2677                 })
2678
2679             url = compat_urlparse.urlunparse(
2680                 parsed_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2681
2682             self._download_webpage(
2683                 url, video_id, f'Marking {label}watched',
2684                 'Unable to mark watched', fatal=False)
2685
2686     @staticmethod
2687     def _extract_urls(webpage):
2688         # Embedded YouTube player
2689         entries = [
2690             unescapeHTML(mobj.group('url'))
2691             for mobj in re.finditer(r'''(?x)
2692             (?:
2693                 <iframe[^>]+?src=|
2694                 data-video-url=|
2695                 <embed[^>]+?src=|
2696                 embedSWF\(?:\s*|
2697                 <object[^>]+data=|
2698                 new\s+SWFObject\(
2699             )
2700             (["\'])
2701                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2702                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2703             \1''', webpage)]
2704
2705         # lazyYT YouTube embed
2706         entries.extend(list(map(
2707             unescapeHTML,
2708             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2709
2710         # Wordpress "YouTube Video Importer" plugin
2711         matches = re.findall(r'''(?x)<div[^>]+
2712             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2713             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2714         entries.extend(m[-1] for m in matches)
2715
2716         return entries
2717
2718     @staticmethod
2719     def _extract_url(webpage):
2720         urls = YoutubeIE._extract_urls(webpage)
2721         return urls[0] if urls else None
2722
2723     @classmethod
2724     def extract_id(cls, url):
2725         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2726         if mobj is None:
2727             raise ExtractorError('Invalid URL: %s' % url)
2728         return mobj.group('id')
2729
2730     def _extract_chapters_from_json(self, data, duration):
2731         chapter_list = traverse_obj(
2732             data, (
2733                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2734                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2735             ), expected_type=list)
2736
2737         return self._extract_chapters(
2738             chapter_list,
2739             chapter_time=lambda chapter: float_or_none(
2740                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2741             chapter_title=lambda chapter: traverse_obj(
2742                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2743             duration=duration)
2744
2745     def _extract_chapters_from_engagement_panel(self, data, duration):
2746         content_list = traverse_obj(
2747             data,
2748             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2749             expected_type=list, default=[])
2750         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2751         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2752
2753         return next(filter(None, (
2754             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2755                                    chapter_time, chapter_title, duration)
2756             for contents in content_list)), [])
2757
2758     def _extract_chapters_from_description(self, description, duration):
2759         return self._extract_chapters(
2760             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
2761             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
2762             duration=duration, strict=False)
2763
2764     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
2765         if not duration:
2766             return
2767         chapter_list = [{
2768             'start_time': chapter_time(chapter),
2769             'title': chapter_title(chapter),
2770         } for chapter in chapter_list or []]
2771         if not strict:
2772             chapter_list.sort(key=lambda c: c['start_time'] or 0)
2773
2774         chapters = [{'start_time': 0, 'title': '<Untitled>'}]
2775         for idx, chapter in enumerate(chapter_list):
2776             if chapter['start_time'] is None or not chapter['title']:
2777                 self.report_warning(f'Incomplete chapter {idx}')
2778             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
2779                 chapters[-1]['end_time'] = chapter['start_time']
2780                 chapters.append(chapter)
2781             else:
2782                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
2783         chapters[-1]['end_time'] = duration
2784         return chapters if len(chapters) > 1 and chapters[1]['start_time'] else chapters[1:]
2785
2786     def _extract_comment(self, comment_renderer, parent=None):
2787         comment_id = comment_renderer.get('commentId')
2788         if not comment_id:
2789             return
2790
2791         text = self._get_text(comment_renderer, 'contentText')
2792
2793         # note: timestamp is an estimate calculated from the current time and time_text
2794         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2795         author = self._get_text(comment_renderer, 'authorText')
2796         author_id = try_get(comment_renderer,
2797                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2798
2799         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2800                                                        lambda x: x['likeCount']), compat_str)) or 0
2801         author_thumbnail = try_get(comment_renderer,
2802                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2803
2804         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2805         is_favorited = 'creatorHeart' in (try_get(
2806             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2807         return {
2808             'id': comment_id,
2809             'text': text,
2810             'timestamp': timestamp,
2811             'time_text': time_text,
2812             'like_count': votes,
2813             'is_favorited': is_favorited,
2814             'author': author,
2815             'author_id': author_id,
2816             'author_thumbnail': author_thumbnail,
2817             'author_is_uploader': author_is_uploader,
2818             'parent': parent or 'root'
2819         }
2820
2821     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2822
2823         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2824
2825         def extract_header(contents):
2826             _continuation = None
2827             for content in contents:
2828                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2829                 expected_comment_count = self._get_count(
2830                     comments_header_renderer, 'countText', 'commentsCount')
2831
2832                 if expected_comment_count:
2833                     tracker['est_total'] = expected_comment_count
2834                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2835                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2836
2837                 sort_menu_item = try_get(
2838                     comments_header_renderer,
2839                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2840                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2841
2842                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2843                 if not _continuation:
2844                     continue
2845
2846                 sort_text = str_or_none(sort_menu_item.get('title'))
2847                 if not sort_text:
2848                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2849                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2850                 break
2851             return _continuation
2852
2853         def extract_thread(contents):
2854             if not parent:
2855                 tracker['current_page_thread'] = 0
2856             for content in contents:
2857                 if not parent and tracker['total_parent_comments'] >= max_parents:
2858                     yield
2859                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2860                 comment_renderer = get_first(
2861                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2862                     expected_type=dict, default={})
2863
2864                 comment = self._extract_comment(comment_renderer, parent)
2865                 if not comment:
2866                     continue
2867
2868                 tracker['running_total'] += 1
2869                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2870                 yield comment
2871
2872                 # Attempt to get the replies
2873                 comment_replies_renderer = try_get(
2874                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2875
2876                 if comment_replies_renderer:
2877                     tracker['current_page_thread'] += 1
2878                     comment_entries_iter = self._comment_entries(
2879                         comment_replies_renderer, ytcfg, video_id,
2880                         parent=comment.get('id'), tracker=tracker)
2881                     yield from itertools.islice(comment_entries_iter, min(
2882                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2883
2884         # Keeps track of counts across recursive calls
2885         if not tracker:
2886             tracker = dict(
2887                 running_total=0,
2888                 est_total=0,
2889                 current_page_thread=0,
2890                 total_parent_comments=0,
2891                 total_reply_comments=0)
2892
2893         # TODO: Deprecated
2894         # YouTube comments have a max depth of 2
2895         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2896         if max_depth:
2897             self._downloader.deprecation_warning(
2898                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2899         if max_depth == 1 and parent:
2900             return
2901
2902         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2903             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2904
2905         continuation = self._extract_continuation(root_continuation_data)
2906
2907         response = None
2908         is_forced_continuation = False
2909         is_first_continuation = parent is None
2910         if is_first_continuation and not continuation:
2911             # Sometimes you can get comments by generating the continuation yourself,
2912             # even if YouTube initially reports them being disabled - e.g. stories comments.
2913             # Note: if the comment section is actually disabled, YouTube may return a response with
2914             # required check_get_keys missing. So we will disable that check initially in this case.
2915             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2916             is_forced_continuation = True
2917
2918         for page_num in itertools.count(0):
2919             if not continuation:
2920                 break
2921             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2922             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2923             if page_num == 0:
2924                 if is_first_continuation:
2925                     note_prefix = 'Downloading comment section API JSON'
2926                 else:
2927                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2928                         tracker['current_page_thread'], comment_prog_str)
2929             else:
2930                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2931                     '       ' if parent else '', ' replies' if parent else '',
2932                     page_num, comment_prog_str)
2933
2934             response = self._extract_response(
2935                 item_id=None, query=continuation,
2936                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2937                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2938             is_forced_continuation = False
2939             continuation_contents = traverse_obj(
2940                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2941
2942             continuation = None
2943             for continuation_section in continuation_contents:
2944                 continuation_items = traverse_obj(
2945                     continuation_section,
2946                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2947                     get_all=False, expected_type=list) or []
2948                 if is_first_continuation:
2949                     continuation = extract_header(continuation_items)
2950                     is_first_continuation = False
2951                     if continuation:
2952                         break
2953                     continue
2954
2955                 for entry in extract_thread(continuation_items):
2956                     if not entry:
2957                         return
2958                     yield entry
2959                 continuation = self._extract_continuation({'contents': continuation_items})
2960                 if continuation:
2961                     break
2962
2963         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2964         if message and not parent and tracker['running_total'] == 0:
2965             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2966
2967     @staticmethod
2968     def _generate_comment_continuation(video_id):
2969         """
2970         Generates initial comment section continuation token from given video id
2971         """
2972         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2973         return base64.b64encode(token.encode()).decode()
2974
2975     def _get_comments(self, ytcfg, video_id, contents, webpage):
2976         """Entry for comment extraction"""
2977         def _real_comment_extract(contents):
2978             renderer = next((
2979                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2980                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2981             yield from self._comment_entries(renderer, ytcfg, video_id)
2982
2983         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2984         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2985
2986     @staticmethod
2987     def _get_checkok_params():
2988         return {'contentCheckOk': True, 'racyCheckOk': True}
2989
2990     @classmethod
2991     def _generate_player_context(cls, sts=None):
2992         context = {
2993             'html5Preference': 'HTML5_PREF_WANTS',
2994         }
2995         if sts is not None:
2996             context['signatureTimestamp'] = sts
2997         return {
2998             'playbackContext': {
2999                 'contentPlaybackContext': context
3000             },
3001             **cls._get_checkok_params()
3002         }
3003
3004     @staticmethod
3005     def _is_agegated(player_response):
3006         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3007             return True
3008
3009         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3010         AGE_GATE_REASONS = (
3011             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3012             'age_verification_required', 'age_check_required',  # status
3013         )
3014         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3015
3016     @staticmethod
3017     def _is_unplayable(player_response):
3018         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3019
3020     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
3021
3022         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3023         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3024         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3025         headers = self.generate_api_headers(
3026             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3027
3028         yt_query = {
3029             'videoId': video_id,
3030             'params': '8AEB'  # enable stories
3031         }
3032         yt_query.update(self._generate_player_context(sts))
3033         return self._extract_response(
3034             item_id=video_id, ep='player', query=yt_query,
3035             ytcfg=player_ytcfg, headers=headers, fatal=True,
3036             default_client=client,
3037             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3038         ) or None
3039
3040     def _get_requested_clients(self, url, smuggled_data):
3041         requested_clients = []
3042         default = ['android', 'web']
3043         allowed_clients = sorted(
3044             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3045             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3046         for client in self._configuration_arg('player_client'):
3047             if client in allowed_clients:
3048                 requested_clients.append(client)
3049             elif client == 'default':
3050                 requested_clients.extend(default)
3051             elif client == 'all':
3052                 requested_clients.extend(allowed_clients)
3053             else:
3054                 self.report_warning(f'Skipping unsupported client {client}')
3055         if not requested_clients:
3056             requested_clients = default
3057
3058         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3059             requested_clients.extend(
3060                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3061
3062         return orderedSet(requested_clients)
3063
3064     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3065         initial_pr = None
3066         if webpage:
3067             initial_pr = self._search_json(
3068                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3069
3070         all_clients = set(clients)
3071         clients = clients[::-1]
3072         prs = []
3073
3074         def append_client(*client_names):
3075             """ Append the first client name that exists but not already used """
3076             for client_name in client_names:
3077                 actual_client = _split_innertube_client(client_name)[0]
3078                 if actual_client in INNERTUBE_CLIENTS:
3079                     if actual_client not in all_clients:
3080                         clients.append(client_name)
3081                         all_clients.add(actual_client)
3082                         return
3083
3084         # Android player_response does not have microFormats which are needed for
3085         # extraction of some data. So we return the initial_pr with formats
3086         # stripped out even if not requested by the user
3087         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3088         if initial_pr:
3089             pr = dict(initial_pr)
3090             pr['streamingData'] = None
3091             prs.append(pr)
3092
3093         last_error = None
3094         tried_iframe_fallback = False
3095         player_url = None
3096         while clients:
3097             client, base_client, variant = _split_innertube_client(clients.pop())
3098             player_ytcfg = master_ytcfg if client == 'web' else {}
3099             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3100                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3101
3102             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3103             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3104             if 'js' in self._configuration_arg('player_skip'):
3105                 require_js_player = False
3106                 player_url = None
3107
3108             if not player_url and not tried_iframe_fallback and require_js_player:
3109                 player_url = self._download_player_url(video_id)
3110                 tried_iframe_fallback = True
3111
3112             try:
3113                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3114                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3115             except ExtractorError as e:
3116                 if last_error:
3117                     self.report_warning(last_error)
3118                 last_error = e
3119                 continue
3120
3121             if pr:
3122                 prs.append(pr)
3123
3124             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3125             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3126                 append_client(f'{base_client}_creator')
3127             elif self._is_agegated(pr):
3128                 if variant == 'tv_embedded':
3129                     append_client(f'{base_client}_embedded')
3130                 elif not variant:
3131                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3132
3133         if last_error:
3134             if not len(prs):
3135                 raise last_error
3136             self.report_warning(last_error)
3137         return prs, player_url
3138
3139     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3140         itags, stream_ids = {}, []
3141         itag_qualities, res_qualities = {}, {}
3142         q = qualities([
3143             # Normally tiny is the smallest video-only formats. But
3144             # audio-only formats with unknown quality may get tagged as tiny
3145             'tiny',
3146             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3147             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3148         ])
3149         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3150
3151         for fmt in streaming_formats:
3152             if fmt.get('targetDurationSec'):
3153                 continue
3154
3155             itag = str_or_none(fmt.get('itag'))
3156             audio_track = fmt.get('audioTrack') or {}
3157             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3158             if stream_id in stream_ids:
3159                 continue
3160
3161             quality = fmt.get('quality')
3162             height = int_or_none(fmt.get('height'))
3163             if quality == 'tiny' or not quality:
3164                 quality = fmt.get('audioQuality', '').lower() or quality
3165             # The 3gp format (17) in android client has a quality of "small",
3166             # but is actually worse than other formats
3167             if itag == '17':
3168                 quality = 'tiny'
3169             if quality:
3170                 if itag:
3171                     itag_qualities[itag] = quality
3172                 if height:
3173                     res_qualities[height] = quality
3174             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3175             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3176             # number of fragment that would subsequently requested with (`&sq=N`)
3177             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3178                 continue
3179
3180             fmt_url = fmt.get('url')
3181             if not fmt_url:
3182                 sc = compat_parse_qs(fmt.get('signatureCipher'))
3183                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3184                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3185                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3186                     continue
3187                 try:
3188                     fmt_url += '&%s=%s' % (
3189                         traverse_obj(sc, ('sp', -1)) or 'signature',
3190                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3191                     )
3192                 except ExtractorError as e:
3193                     self.report_warning('Signature extraction failed: Some formats may be missing', only_once=True)
3194                     self.write_debug(e, only_once=True)
3195                     continue
3196
3197             query = parse_qs(fmt_url)
3198             throttled = False
3199             if query.get('n'):
3200                 try:
3201                     fmt_url = update_url_query(fmt_url, {
3202                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3203                 except ExtractorError as e:
3204                     self.report_warning(
3205                         'nsig extraction failed: You may experience throttling for some formats\n'
3206                         f'n = {query["n"][0]} ; player = {player_url}', only_once=True)
3207                     self.write_debug(e, only_once=True)
3208                     throttled = True
3209
3210             if itag:
3211                 itags[itag] = 'https'
3212                 stream_ids.append(stream_id)
3213
3214             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3215             language_preference = (
3216                 10 if audio_track.get('audioIsDefault') and 10
3217                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3218                 else -1)
3219             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3220             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3221             # Make sure to avoid false positives with small duration differences.
3222             # Eg: __2ABJjxzNo, ySuUZEjARPY
3223             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3224             if is_damaged:
3225                 self.report_warning(
3226                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3227             dct = {
3228                 'asr': int_or_none(fmt.get('audioSampleRate')),
3229                 'filesize': int_or_none(fmt.get('contentLength')),
3230                 'format_id': itag,
3231                 'format_note': join_nonempty(
3232                     '%s%s' % (audio_track.get('displayName') or '',
3233                               ' (default)' if language_preference > 0 else ''),
3234                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3235                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3236                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3237                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3238                 'fps': int_or_none(fmt.get('fps')) or None,
3239                 'height': height,
3240                 'quality': q(quality),
3241                 'has_drm': bool(fmt.get('drmFamilies')),
3242                 'tbr': tbr,
3243                 'url': fmt_url,
3244                 'width': int_or_none(fmt.get('width')),
3245                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3246                                           'desc' if language_preference < -1 else ''),
3247                 'language_preference': language_preference,
3248                 # Strictly de-prioritize damaged and 3gp formats
3249                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3250             }
3251             mime_mobj = re.match(
3252                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3253             if mime_mobj:
3254                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3255                 dct.update(parse_codecs(mime_mobj.group(2)))
3256             no_audio = dct.get('acodec') == 'none'
3257             no_video = dct.get('vcodec') == 'none'
3258             if no_audio:
3259                 dct['vbr'] = tbr
3260             if no_video:
3261                 dct['abr'] = tbr
3262             if no_audio or no_video:
3263                 dct['downloader_options'] = {
3264                     # Youtube throttles chunks >~10M
3265                     'http_chunk_size': 10485760,
3266                 }
3267                 if dct.get('ext'):
3268                     dct['container'] = dct['ext'] + '_dash'
3269             yield dct
3270
3271         live_from_start = is_live and self.get_param('live_from_start')
3272         skip_manifests = self._configuration_arg('skip')
3273         if not self.get_param('youtube_include_hls_manifest', True):
3274             skip_manifests.append('hls')
3275         if not self.get_param('youtube_include_dash_manifest', True):
3276             skip_manifests.append('dash')
3277         get_dash = 'dash' not in skip_manifests and (
3278             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3279         get_hls = not live_from_start and 'hls' not in skip_manifests
3280
3281         def process_manifest_format(f, proto, itag):
3282             if itag in itags:
3283                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3284                     return False
3285                 itag = f'{itag}-{proto}'
3286             if itag:
3287                 f['format_id'] = itag
3288                 itags[itag] = proto
3289
3290             f['quality'] = next((
3291                 q(qdict[val])
3292                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3293                 if val in qdict), -1)
3294             return True
3295
3296         for sd in streaming_data:
3297             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3298             if hls_manifest_url:
3299                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3300                     if process_manifest_format(f, 'hls', self._search_regex(
3301                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3302                         yield f
3303
3304             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3305             if dash_manifest_url:
3306                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3307                     if process_manifest_format(f, 'dash', f['format_id']):
3308                         f['filesize'] = int_or_none(self._search_regex(
3309                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3310                         if live_from_start:
3311                             f['is_from_start'] = True
3312
3313                         yield f
3314
3315     def _extract_storyboard(self, player_responses, duration):
3316         spec = get_first(
3317             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3318         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3319         if not base_url:
3320             return
3321         L = len(spec) - 1
3322         for i, args in enumerate(spec):
3323             args = args.split('#')
3324             counts = list(map(int_or_none, args[:5]))
3325             if len(args) != 8 or not all(counts):
3326                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3327                 continue
3328             width, height, frame_count, cols, rows = counts
3329             N, sigh = args[6:]
3330
3331             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3332             fragment_count = frame_count / (cols * rows)
3333             fragment_duration = duration / fragment_count
3334             yield {
3335                 'format_id': f'sb{i}',
3336                 'format_note': 'storyboard',
3337                 'ext': 'mhtml',
3338                 'protocol': 'mhtml',
3339                 'acodec': 'none',
3340                 'vcodec': 'none',
3341                 'url': url,
3342                 'width': width,
3343                 'height': height,
3344                 'fragments': [{
3345                     'url': url.replace('$M', str(j)),
3346                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3347                 } for j in range(math.ceil(fragment_count))],
3348             }
3349
3350     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3351         webpage = None
3352         if 'webpage' not in self._configuration_arg('player_skip'):
3353             webpage = self._download_webpage(
3354                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3355
3356         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3357
3358         player_responses, player_url = self._extract_player_responses(
3359             self._get_requested_clients(url, smuggled_data),
3360             video_id, webpage, master_ytcfg)
3361
3362         return webpage, master_ytcfg, player_responses, player_url
3363
3364     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3365         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3366         is_live = get_first(video_details, 'isLive')
3367         if is_live is None:
3368             is_live = get_first(live_broadcast_details, 'isLiveNow')
3369
3370         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3371         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3372
3373         return live_broadcast_details, is_live, streaming_data, formats
3374
3375     def _real_extract(self, url):
3376         url, smuggled_data = unsmuggle_url(url, {})
3377         video_id = self._match_id(url)
3378
3379         base_url = self.http_scheme() + '//www.youtube.com/'
3380         webpage_url = base_url + 'watch?v=' + video_id
3381
3382         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3383
3384         playability_statuses = traverse_obj(
3385             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3386
3387         trailer_video_id = get_first(
3388             playability_statuses,
3389             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3390             expected_type=str)
3391         if trailer_video_id:
3392             return self.url_result(
3393                 trailer_video_id, self.ie_key(), trailer_video_id)
3394
3395         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3396                        if webpage else (lambda x: None))
3397
3398         video_details = traverse_obj(
3399             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3400         microformats = traverse_obj(
3401             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3402             expected_type=dict, default=[])
3403         video_title = (
3404             get_first(video_details, 'title')
3405             or self._get_text(microformats, (..., 'title'))
3406             or search_meta(['og:title', 'twitter:title', 'title']))
3407         video_description = get_first(video_details, 'shortDescription')
3408
3409         multifeed_metadata_list = get_first(
3410             player_responses,
3411             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3412             expected_type=str)
3413         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3414             if self.get_param('noplaylist'):
3415                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3416             else:
3417                 entries = []
3418                 feed_ids = []
3419                 for feed in multifeed_metadata_list.split(','):
3420                     # Unquote should take place before split on comma (,) since textual
3421                     # fields may contain comma as well (see
3422                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3423                     feed_data = compat_parse_qs(
3424                         compat_urllib_parse_unquote_plus(feed))
3425
3426                     def feed_entry(name):
3427                         return try_get(
3428                             feed_data, lambda x: x[name][0], compat_str)
3429
3430                     feed_id = feed_entry('id')
3431                     if not feed_id:
3432                         continue
3433                     feed_title = feed_entry('title')
3434                     title = video_title
3435                     if feed_title:
3436                         title += ' (%s)' % feed_title
3437                     entries.append({
3438                         '_type': 'url_transparent',
3439                         'ie_key': 'Youtube',
3440                         'url': smuggle_url(
3441                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3442                             {'force_singlefeed': True}),
3443                         'title': title,
3444                     })
3445                     feed_ids.append(feed_id)
3446                 self.to_screen(
3447                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3448                     % (', '.join(feed_ids), video_id))
3449                 return self.playlist_result(
3450                     entries, video_id, video_title, video_description)
3451
3452         duration = int_or_none(
3453             get_first(video_details, 'lengthSeconds')
3454             or get_first(microformats, 'lengthSeconds')
3455             or parse_duration(search_meta('duration'))) or None
3456
3457         if get_first(video_details, 'isPostLiveDvr'):
3458             self.write_debug('Video is in Post-Live Manifestless mode')
3459             if duration or 0 > 4 * 3600:
3460                 self.report_warning(
3461                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3462                     'This is a known issue and patches are welcome')
3463
3464         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3465             video_id, microformats, video_details, player_responses, player_url, duration)
3466
3467         if not formats:
3468             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3469                 self.report_drm(video_id)
3470             pemr = get_first(
3471                 playability_statuses,
3472                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3473             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3474             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3475             if subreason:
3476                 if subreason == 'The uploader has not made this video available in your country.':
3477                     countries = get_first(microformats, 'availableCountries')
3478                     if not countries:
3479                         regions_allowed = search_meta('regionsAllowed')
3480                         countries = regions_allowed.split(',') if regions_allowed else None
3481                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3482                 reason += f'. {subreason}'
3483             if reason:
3484                 self.raise_no_formats(reason, expected=True)
3485
3486         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3487         if not keywords and webpage:
3488             keywords = [
3489                 unescapeHTML(m.group('content'))
3490                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3491         for keyword in keywords:
3492             if keyword.startswith('yt:stretch='):
3493                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3494                 if mobj:
3495                     # NB: float is intentional for forcing float division
3496                     w, h = (float(v) for v in mobj.groups())
3497                     if w > 0 and h > 0:
3498                         ratio = w / h
3499                         for f in formats:
3500                             if f.get('vcodec') != 'none':
3501                                 f['stretched_ratio'] = ratio
3502                         break
3503         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3504         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3505         if thumbnail_url:
3506             thumbnails.append({
3507                 'url': thumbnail_url,
3508             })
3509         original_thumbnails = thumbnails.copy()
3510
3511         # The best resolution thumbnails sometimes does not appear in the webpage
3512         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3513         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3514         thumbnail_names = [
3515             # While the *1,*2,*3 thumbnails are just below their correspnding "*default" variants
3516             # in resolution, these are not the custom thumbnail. So de-prioritize them
3517             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3518             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3519         ]
3520         n_thumbnail_names = len(thumbnail_names)
3521         thumbnails.extend({
3522             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3523                 video_id=video_id, name=name, ext=ext,
3524                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3525         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3526         for thumb in thumbnails:
3527             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3528             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3529         self._remove_duplicate_formats(thumbnails)
3530         self._downloader._sort_thumbnails(original_thumbnails)
3531
3532         category = get_first(microformats, 'category') or search_meta('genre')
3533         channel_id = str_or_none(
3534             get_first(video_details, 'channelId')
3535             or get_first(microformats, 'externalChannelId')
3536             or search_meta('channelId'))
3537         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3538
3539         live_content = get_first(video_details, 'isLiveContent')
3540         is_upcoming = get_first(video_details, 'isUpcoming')
3541         if is_live is None:
3542             if is_upcoming or live_content is False:
3543                 is_live = False
3544         if is_upcoming is None and (live_content or is_live):
3545             is_upcoming = False
3546         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3547         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3548         if not duration and live_end_time and live_start_time:
3549             duration = live_end_time - live_start_time
3550
3551         if is_live and self.get_param('live_from_start'):
3552             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3553
3554         formats.extend(self._extract_storyboard(player_responses, duration))
3555
3556         # Source is given priority since formats that throttle are given lower source_preference
3557         # When throttling issue is fully fixed, remove this
3558         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3559
3560         info = {
3561             'id': video_id,
3562             'title': video_title,
3563             'formats': formats,
3564             'thumbnails': thumbnails,
3565             # The best thumbnail that we are sure exists. Prevents unnecessary
3566             # URL checking if user don't care about getting the best possible thumbnail
3567             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3568             'description': video_description,
3569             'uploader': get_first(video_details, 'author'),
3570             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3571             'uploader_url': owner_profile_url,
3572             'channel_id': channel_id,
3573             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3574             'duration': duration,
3575             'view_count': int_or_none(
3576                 get_first((video_details, microformats), (..., 'viewCount'))
3577                 or search_meta('interactionCount')),
3578             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3579             'age_limit': 18 if (
3580                 get_first(microformats, 'isFamilySafe') is False
3581                 or search_meta('isFamilyFriendly') == 'false'
3582                 or search_meta('og:restrictions:age') == '18+') else 0,
3583             'webpage_url': webpage_url,
3584             'categories': [category] if category else None,
3585             'tags': keywords,
3586             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3587             'is_live': is_live,
3588             'was_live': (False if is_live or is_upcoming or live_content is False
3589                          else None if is_live is None or is_upcoming is None
3590                          else live_content),
3591             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3592             'release_timestamp': live_start_time,
3593         }
3594
3595         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3596         if pctr:
3597             def get_lang_code(track):
3598                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3599                         or track.get('languageCode'))
3600
3601             # Converted into dicts to remove duplicates
3602             captions = {
3603                 get_lang_code(sub): sub
3604                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3605             translation_languages = {
3606                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3607                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3608
3609             def process_language(container, base_url, lang_code, sub_name, query):
3610                 lang_subs = container.setdefault(lang_code, [])
3611                 for fmt in self._SUBTITLE_FORMATS:
3612                     query.update({
3613                         'fmt': fmt,
3614                     })
3615                     lang_subs.append({
3616                         'ext': fmt,
3617                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3618                         'name': sub_name,
3619                     })
3620
3621             subtitles, automatic_captions = {}, {}
3622             for lang_code, caption_track in captions.items():
3623                 base_url = caption_track.get('baseUrl')
3624                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3625                 if not base_url:
3626                     continue
3627                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3628                 if caption_track.get('kind') != 'asr':
3629                     if not lang_code:
3630                         continue
3631                     process_language(
3632                         subtitles, base_url, lang_code, lang_name, {})
3633                     if not caption_track.get('isTranslatable'):
3634                         continue
3635                 for trans_code, trans_name in translation_languages.items():
3636                     if not trans_code:
3637                         continue
3638                     orig_trans_code = trans_code
3639                     if caption_track.get('kind') != 'asr':
3640                         if 'translated_subs' in self._configuration_arg('skip'):
3641                             continue
3642                         trans_code += f'-{lang_code}'
3643                         trans_name += format_field(lang_name, None, ' from %s')
3644                     # Add an "-orig" label to the original language so that it can be distinguished.
3645                     # The subs are returned without "-orig" as well for compatibility
3646                     if lang_code == f'a-{orig_trans_code}':
3647                         process_language(
3648                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3649                     # Setting tlang=lang returns damaged subtitles.
3650                     process_language(automatic_captions, base_url, trans_code, trans_name,
3651                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3652             info['automatic_captions'] = automatic_captions
3653             info['subtitles'] = subtitles
3654
3655         parsed_url = compat_urllib_parse_urlparse(url)
3656         for component in [parsed_url.fragment, parsed_url.query]:
3657             query = compat_parse_qs(component)
3658             for k, v in query.items():
3659                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3660                     d_k += '_time'
3661                     if d_k not in info and k in s_ks:
3662                         info[d_k] = parse_duration(query[k][0])
3663
3664         # Youtube Music Auto-generated description
3665         if video_description:
3666             mobj = re.search(
3667                 r'''(?xs)
3668                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3669                     (?P<album>[^\n]+)
3670                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3671                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3672                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3673                     .+\nAuto-generated\ by\ YouTube\.\s*$
3674                 ''', video_description)
3675             if mobj:
3676                 release_year = mobj.group('release_year')
3677                 release_date = mobj.group('release_date')
3678                 if release_date:
3679                     release_date = release_date.replace('-', '')
3680                     if not release_year:
3681                         release_year = release_date[:4]
3682                 info.update({
3683                     'album': mobj.group('album'.strip()),
3684                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3685                     'track': mobj.group('track').strip(),
3686                     'release_date': release_date,
3687                     'release_year': int_or_none(release_year),
3688                 })
3689
3690         initial_data = None
3691         if webpage:
3692             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
3693         if not initial_data:
3694             query = {'videoId': video_id}
3695             query.update(self._get_checkok_params())
3696             initial_data = self._extract_response(
3697                 item_id=video_id, ep='next', fatal=False,
3698                 ytcfg=master_ytcfg, query=query,
3699                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3700                 note='Downloading initial data API JSON')
3701
3702         info['comment_count'] = traverse_obj(initial_data, (
3703             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
3704             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
3705         ), (
3706             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
3707             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
3708         ), expected_type=int_or_none, get_all=False)
3709
3710         try:  # This will error if there is no livechat
3711             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3712         except (KeyError, IndexError, TypeError):
3713             pass
3714         else:
3715             info.setdefault('subtitles', {})['live_chat'] = [{
3716                 # url is needed to set cookies
3717                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
3718                 'video_id': video_id,
3719                 'ext': 'json',
3720                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3721             }]
3722
3723         if initial_data:
3724             info['chapters'] = (
3725                 self._extract_chapters_from_json(initial_data, duration)
3726                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3727                 or self._extract_chapters_from_description(video_description, duration)
3728                 or None)
3729
3730         contents = traverse_obj(
3731             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3732             expected_type=list, default=[])
3733
3734         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3735         if vpir:
3736             stl = vpir.get('superTitleLink')
3737             if stl:
3738                 stl = self._get_text(stl)
3739                 if try_get(
3740                         vpir,
3741                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3742                     info['location'] = stl
3743                 else:
3744                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3745                     if mobj:
3746                         info.update({
3747                             'series': mobj.group(1),
3748                             'season_number': int(mobj.group(2)),
3749                             'episode_number': int(mobj.group(3)),
3750                         })
3751             for tlb in (try_get(
3752                     vpir,
3753                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3754                     list) or []):
3755                 tbr = tlb.get('toggleButtonRenderer') or {}
3756                 for getter, regex in [(
3757                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3758                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3759                             lambda x: x['accessibility'],
3760                             lambda x: x['accessibilityData']['accessibilityData'],
3761                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3762                     label = (try_get(tbr, getter, dict) or {}).get('label')
3763                     if label:
3764                         mobj = re.match(regex, label)
3765                         if mobj:
3766                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3767                             break
3768             sbr_tooltip = try_get(
3769                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3770             if sbr_tooltip:
3771                 like_count, dislike_count = sbr_tooltip.split(' / ')
3772                 info.update({
3773                     'like_count': str_to_int(like_count),
3774                     'dislike_count': str_to_int(dislike_count),
3775                 })
3776         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3777         if vsir:
3778             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3779             info.update({
3780                 'channel': self._get_text(vor, 'title'),
3781                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3782
3783             rows = try_get(
3784                 vsir,
3785                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3786                 list) or []
3787             multiple_songs = False
3788             for row in rows:
3789                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3790                     multiple_songs = True
3791                     break
3792             for row in rows:
3793                 mrr = row.get('metadataRowRenderer') or {}
3794                 mrr_title = mrr.get('title')
3795                 if not mrr_title:
3796                     continue
3797                 mrr_title = self._get_text(mrr, 'title')
3798                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3799                 if mrr_title == 'License':
3800                     info['license'] = mrr_contents_text
3801                 elif not multiple_songs:
3802                     if mrr_title == 'Album':
3803                         info['album'] = mrr_contents_text
3804                     elif mrr_title == 'Artist':
3805                         info['artist'] = mrr_contents_text
3806                     elif mrr_title == 'Song':
3807                         info['track'] = mrr_contents_text
3808
3809         fallbacks = {
3810             'channel': 'uploader',
3811             'channel_id': 'uploader_id',
3812             'channel_url': 'uploader_url',
3813         }
3814
3815         # The upload date for scheduled, live and past live streams / premieres in microformats
3816         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3817         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3818         upload_date = (
3819             unified_strdate(get_first(microformats, 'uploadDate'))
3820             or unified_strdate(search_meta('uploadDate')))
3821         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3822             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3823         info['upload_date'] = upload_date
3824
3825         for to, frm in fallbacks.items():
3826             if not info.get(to):
3827                 info[to] = info.get(frm)
3828
3829         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3830             v = info.get(s_k)
3831             if v:
3832                 info[d_k] = v
3833
3834         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3835         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3836         is_membersonly = None
3837         is_premium = None
3838         if initial_data and is_private is not None:
3839             is_membersonly = False
3840             is_premium = False
3841             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3842             badge_labels = set()
3843             for content in contents:
3844                 if not isinstance(content, dict):
3845                     continue
3846                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3847             for badge_label in badge_labels:
3848                 if badge_label.lower() == 'members only':
3849                     is_membersonly = True
3850                 elif badge_label.lower() == 'premium':
3851                     is_premium = True
3852                 elif badge_label.lower() == 'unlisted':
3853                     is_unlisted = True
3854
3855         info['availability'] = self._availability(
3856             is_private=is_private,
3857             needs_premium=is_premium,
3858             needs_subscription=is_membersonly,
3859             needs_auth=info['age_limit'] >= 18,
3860             is_unlisted=None if is_private is None else is_unlisted)
3861
3862         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3863
3864         self.mark_watched(video_id, player_responses)
3865
3866         return info
3867
3868
3869 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3870
3871     @staticmethod
3872     def passthrough_smuggled_data(func):
3873         def _smuggle(entries, smuggled_data):
3874             for entry in entries:
3875                 # TODO: Convert URL to music.youtube instead.
3876                 # Do we need to passthrough any other smuggled_data?
3877                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3878                 yield entry
3879
3880         @functools.wraps(func)
3881         def wrapper(self, url):
3882             url, smuggled_data = unsmuggle_url(url, {})
3883             if self.is_music_url(url):
3884                 smuggled_data['is_music_url'] = True
3885             info_dict = func(self, url, smuggled_data)
3886             if smuggled_data and info_dict.get('entries'):
3887                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3888             return info_dict
3889         return wrapper
3890
3891     def _extract_channel_id(self, webpage):
3892         channel_id = self._html_search_meta(
3893             'channelId', webpage, 'channel id', default=None)
3894         if channel_id:
3895             return channel_id
3896         channel_url = self._html_search_meta(
3897             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3898              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3899              'twitter:app:url:googleplay'), webpage, 'channel url')
3900         return self._search_regex(
3901             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3902             channel_url, 'channel id')
3903
3904     @staticmethod
3905     def _extract_basic_item_renderer(item):
3906         # Modified from _extract_grid_item_renderer
3907         known_basic_renderers = (
3908             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3909         )
3910         for key, renderer in item.items():
3911             if not isinstance(renderer, dict):
3912                 continue
3913             elif key in known_basic_renderers:
3914                 return renderer
3915             elif key.startswith('grid') and key.endswith('Renderer'):
3916                 return renderer
3917
3918     def _grid_entries(self, grid_renderer):
3919         for item in grid_renderer['items']:
3920             if not isinstance(item, dict):
3921                 continue
3922             renderer = self._extract_basic_item_renderer(item)
3923             if not isinstance(renderer, dict):
3924                 continue
3925             title = self._get_text(renderer, 'title')
3926
3927             # playlist
3928             playlist_id = renderer.get('playlistId')
3929             if playlist_id:
3930                 yield self.url_result(
3931                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3932                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3933                     video_title=title)
3934                 continue
3935             # video
3936             video_id = renderer.get('videoId')
3937             if video_id:
3938                 yield self._extract_video(renderer)
3939                 continue
3940             # channel
3941             channel_id = renderer.get('channelId')
3942             if channel_id:
3943                 yield self.url_result(
3944                     'https://www.youtube.com/channel/%s' % channel_id,
3945                     ie=YoutubeTabIE.ie_key(), video_title=title)
3946                 continue
3947             # generic endpoint URL support
3948             ep_url = urljoin('https://www.youtube.com/', try_get(
3949                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3950                 compat_str))
3951             if ep_url:
3952                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3953                     if ie.suitable(ep_url):
3954                         yield self.url_result(
3955                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3956                         break
3957
3958     def _music_reponsive_list_entry(self, renderer):
3959         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3960         if video_id:
3961             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3962                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3963         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3964         if playlist_id:
3965             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3966             if video_id:
3967                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3968                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3969             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3970                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3971         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3972         if browse_id:
3973             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3974                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3975
3976     def _shelf_entries_from_content(self, shelf_renderer):
3977         content = shelf_renderer.get('content')
3978         if not isinstance(content, dict):
3979             return
3980         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3981         if renderer:
3982             # TODO: add support for nested playlists so each shelf is processed
3983             # as separate playlist
3984             # TODO: this includes only first N items
3985             yield from self._grid_entries(renderer)
3986         renderer = content.get('horizontalListRenderer')
3987         if renderer:
3988             # TODO
3989             pass
3990
3991     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3992         ep = try_get(
3993             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3994             compat_str)
3995         shelf_url = urljoin('https://www.youtube.com', ep)
3996         if shelf_url:
3997             # Skipping links to another channels, note that checking for
3998             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3999             # will not work
4000             if skip_channels and '/channels?' in shelf_url:
4001                 return
4002             title = self._get_text(shelf_renderer, 'title')
4003             yield self.url_result(shelf_url, video_title=title)
4004         # Shelf may not contain shelf URL, fallback to extraction from content
4005         yield from self._shelf_entries_from_content(shelf_renderer)
4006
4007     def _playlist_entries(self, video_list_renderer):
4008         for content in video_list_renderer['contents']:
4009             if not isinstance(content, dict):
4010                 continue
4011             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4012             if not isinstance(renderer, dict):
4013                 continue
4014             video_id = renderer.get('videoId')
4015             if not video_id:
4016                 continue
4017             yield self._extract_video(renderer)
4018
4019     def _rich_entries(self, rich_grid_renderer):
4020         renderer = try_get(
4021             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
4022         video_id = renderer.get('videoId')
4023         if not video_id:
4024             return
4025         yield self._extract_video(renderer)
4026
4027     def _video_entry(self, video_renderer):
4028         video_id = video_renderer.get('videoId')
4029         if video_id:
4030             return self._extract_video(video_renderer)
4031
4032     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4033         url = urljoin('https://youtube.com', traverse_obj(
4034             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4035         if url:
4036             return self.url_result(
4037                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4038
4039     def _post_thread_entries(self, post_thread_renderer):
4040         post_renderer = try_get(
4041             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4042         if not post_renderer:
4043             return
4044         # video attachment
4045         video_renderer = try_get(
4046             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4047         video_id = video_renderer.get('videoId')
4048         if video_id:
4049             entry = self._extract_video(video_renderer)
4050             if entry:
4051                 yield entry
4052         # playlist attachment
4053         playlist_id = try_get(
4054             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
4055         if playlist_id:
4056             yield self.url_result(
4057                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4058                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4059         # inline video links
4060         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4061         for run in runs:
4062             if not isinstance(run, dict):
4063                 continue
4064             ep_url = try_get(
4065                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
4066             if not ep_url:
4067                 continue
4068             if not YoutubeIE.suitable(ep_url):
4069                 continue
4070             ep_video_id = YoutubeIE._match_id(ep_url)
4071             if video_id == ep_video_id:
4072                 continue
4073             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4074
4075     def _post_thread_continuation_entries(self, post_thread_continuation):
4076         contents = post_thread_continuation.get('contents')
4077         if not isinstance(contents, list):
4078             return
4079         for content in contents:
4080             renderer = content.get('backstagePostThreadRenderer')
4081             if isinstance(renderer, dict):
4082                 yield from self._post_thread_entries(renderer)
4083                 continue
4084             renderer = content.get('videoRenderer')
4085             if isinstance(renderer, dict):
4086                 yield self._video_entry(renderer)
4087
4088     r''' # unused
4089     def _rich_grid_entries(self, contents):
4090         for content in contents:
4091             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4092             if video_renderer:
4093                 entry = self._video_entry(video_renderer)
4094                 if entry:
4095                     yield entry
4096     '''
4097
4098     def _extract_entries(self, parent_renderer, continuation_list):
4099         # continuation_list is modified in-place with continuation_list = [continuation_token]
4100         continuation_list[:] = [None]
4101         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4102         for content in contents:
4103             if not isinstance(content, dict):
4104                 continue
4105             is_renderer = traverse_obj(
4106                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4107                 expected_type=dict)
4108             if not is_renderer:
4109                 renderer = content.get('richItemRenderer')
4110                 if renderer:
4111                     for entry in self._rich_entries(renderer):
4112                         yield entry
4113                     continuation_list[0] = self._extract_continuation(parent_renderer)
4114                 continue
4115             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4116             for isr_content in isr_contents:
4117                 if not isinstance(isr_content, dict):
4118                     continue
4119
4120                 known_renderers = {
4121                     'playlistVideoListRenderer': self._playlist_entries,
4122                     'gridRenderer': self._grid_entries,
4123                     'reelShelfRenderer': self._grid_entries,
4124                     'shelfRenderer': self._shelf_entries,
4125                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4126                     'backstagePostThreadRenderer': self._post_thread_entries,
4127                     'videoRenderer': lambda x: [self._video_entry(x)],
4128                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4129                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4130                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4131                 }
4132                 for key, renderer in isr_content.items():
4133                     if key not in known_renderers:
4134                         continue
4135                     for entry in known_renderers[key](renderer):
4136                         if entry:
4137                             yield entry
4138                     continuation_list[0] = self._extract_continuation(renderer)
4139                     break
4140
4141             if not continuation_list[0]:
4142                 continuation_list[0] = self._extract_continuation(is_renderer)
4143
4144         if not continuation_list[0]:
4145             continuation_list[0] = self._extract_continuation(parent_renderer)
4146
4147     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4148         continuation_list = [None]
4149         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4150         tab_content = try_get(tab, lambda x: x['content'], dict)
4151         if not tab_content:
4152             return
4153         parent_renderer = (
4154             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4155             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4156         yield from extract_entries(parent_renderer)
4157         continuation = continuation_list[0]
4158
4159         for page_num in itertools.count(1):
4160             if not continuation:
4161                 break
4162             headers = self.generate_api_headers(
4163                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4164             response = self._extract_response(
4165                 item_id=f'{item_id} page {page_num}',
4166                 query=continuation, headers=headers, ytcfg=ytcfg,
4167                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4168
4169             if not response:
4170                 break
4171             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4172             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4173             visitor_data = self._extract_visitor_data(response) or visitor_data
4174
4175             known_continuation_renderers = {
4176                 'playlistVideoListContinuation': self._playlist_entries,
4177                 'gridContinuation': self._grid_entries,
4178                 'itemSectionContinuation': self._post_thread_continuation_entries,
4179                 'sectionListContinuation': extract_entries,  # for feeds
4180             }
4181             continuation_contents = try_get(
4182                 response, lambda x: x['continuationContents'], dict) or {}
4183             continuation_renderer = None
4184             for key, value in continuation_contents.items():
4185                 if key not in known_continuation_renderers:
4186                     continue
4187                 continuation_renderer = value
4188                 continuation_list = [None]
4189                 yield from known_continuation_renderers[key](continuation_renderer)
4190                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4191                 break
4192             if continuation_renderer:
4193                 continue
4194
4195             known_renderers = {
4196                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4197                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4198                 'gridVideoRenderer': (self._grid_entries, 'items'),
4199                 'gridChannelRenderer': (self._grid_entries, 'items'),
4200                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4201                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4202                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4203                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4204             }
4205             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4206             continuation_items = try_get(
4207                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4208             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4209             video_items_renderer = None
4210             for key, value in continuation_item.items():
4211                 if key not in known_renderers:
4212                     continue
4213                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4214                 continuation_list = [None]
4215                 yield from known_renderers[key][0](video_items_renderer)
4216                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4217                 break
4218             if video_items_renderer:
4219                 continue
4220             break
4221
4222     @staticmethod
4223     def _extract_selected_tab(tabs, fatal=True):
4224         for tab in tabs:
4225             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4226             if renderer.get('selected') is True:
4227                 return renderer
4228         else:
4229             if fatal:
4230                 raise ExtractorError('Unable to find selected tab')
4231
4232     def _extract_uploader(self, data):
4233         uploader = {}
4234         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4235         owner = try_get(
4236             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4237         if owner:
4238             owner_text = owner.get('text')
4239             uploader['uploader'] = self._search_regex(
4240                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4241             uploader['uploader_id'] = try_get(
4242                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
4243             uploader['uploader_url'] = urljoin(
4244                 'https://www.youtube.com/',
4245                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
4246         return {k: v for k, v in uploader.items() if v is not None}
4247
4248     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4249         playlist_id = title = description = channel_url = channel_name = channel_id = None
4250         tags = []
4251
4252         selected_tab = self._extract_selected_tab(tabs)
4253         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4254         renderer = try_get(
4255             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4256         if renderer:
4257             channel_name = renderer.get('title')
4258             channel_url = renderer.get('channelUrl')
4259             channel_id = renderer.get('externalId')
4260         else:
4261             renderer = try_get(
4262                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4263
4264         if renderer:
4265             title = renderer.get('title')
4266             description = renderer.get('description', '')
4267             playlist_id = channel_id
4268             tags = renderer.get('keywords', '').split()
4269
4270         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4271         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4272         def _get_uncropped(url):
4273             return url_or_none((url or '').split('=')[0] + '=s0')
4274
4275         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4276         if avatar_thumbnails:
4277             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4278             if uncropped_avatar:
4279                 avatar_thumbnails.append({
4280                     'url': uncropped_avatar,
4281                     'id': 'avatar_uncropped',
4282                     'preference': 1
4283                 })
4284
4285         channel_banners = self._extract_thumbnails(
4286             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4287         for banner in channel_banners:
4288             banner['preference'] = -10
4289
4290         if channel_banners:
4291             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4292             if uncropped_banner:
4293                 channel_banners.append({
4294                     'url': uncropped_banner,
4295                     'id': 'banner_uncropped',
4296                     'preference': -5
4297                 })
4298
4299         primary_thumbnails = self._extract_thumbnails(
4300             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4301
4302         if playlist_id is None:
4303             playlist_id = item_id
4304
4305         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4306         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4307         if title is None:
4308             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4309         title += format_field(selected_tab, 'title', ' - %s')
4310         title += format_field(selected_tab, 'expandedText', ' - %s')
4311
4312         metadata = {
4313             'playlist_id': playlist_id,
4314             'playlist_title': title,
4315             'playlist_description': description,
4316             'uploader': channel_name,
4317             'uploader_id': channel_id,
4318             'uploader_url': channel_url,
4319             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4320             'tags': tags,
4321             'view_count': self._get_count(playlist_stats, 1),
4322             'availability': self._extract_availability(data),
4323             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4324             'playlist_count': self._get_count(playlist_stats, 0),
4325             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4326         }
4327         if not channel_id:
4328             metadata.update(self._extract_uploader(data))
4329         metadata.update({
4330             'channel': metadata['uploader'],
4331             'channel_id': metadata['uploader_id'],
4332             'channel_url': metadata['uploader_url']})
4333         return self.playlist_result(
4334             self._entries(
4335                 selected_tab, playlist_id, ytcfg,
4336                 self._extract_account_syncid(ytcfg, data),
4337                 self._extract_visitor_data(data, ytcfg)),
4338             **metadata)
4339
4340     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4341         first_id = last_id = response = None
4342         for page_num in itertools.count(1):
4343             videos = list(self._playlist_entries(playlist))
4344             if not videos:
4345                 return
4346             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4347             if start >= len(videos):
4348                 return
4349             yield from videos[start:]
4350             first_id = first_id or videos[0]['id']
4351             last_id = videos[-1]['id']
4352             watch_endpoint = try_get(
4353                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4354             headers = self.generate_api_headers(
4355                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4356                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4357             query = {
4358                 'playlistId': playlist_id,
4359                 'videoId': watch_endpoint.get('videoId') or last_id,
4360                 'index': watch_endpoint.get('index') or len(videos),
4361                 'params': watch_endpoint.get('params') or 'OAE%3D'
4362             }
4363             response = self._extract_response(
4364                 item_id='%s page %d' % (playlist_id, page_num),
4365                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4366                 check_get_keys='contents'
4367             )
4368             playlist = try_get(
4369                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4370
4371     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4372         title = playlist.get('title') or try_get(
4373             data, lambda x: x['titleText']['simpleText'], compat_str)
4374         playlist_id = playlist.get('playlistId') or item_id
4375
4376         # Delegating everything except mix playlists to regular tab-based playlist URL
4377         playlist_url = urljoin(url, try_get(
4378             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4379             compat_str))
4380
4381         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4382         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4383         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4384
4385         if playlist_url and playlist_url != url and not is_known_unviewable:
4386             return self.url_result(
4387                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4388                 video_title=title)
4389
4390         return self.playlist_result(
4391             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4392             playlist_id=playlist_id, playlist_title=title)
4393
4394     def _extract_availability(self, data):
4395         """
4396         Gets the availability of a given playlist/tab.
4397         Note: Unless YouTube tells us explicitly, we do not assume it is public
4398         @param data: response
4399         """
4400         is_private = is_unlisted = None
4401         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4402         badge_labels = self._extract_badges(renderer)
4403
4404         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4405         privacy_dropdown_entries = try_get(
4406             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4407         for renderer_dict in privacy_dropdown_entries:
4408             is_selected = try_get(
4409                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4410             if not is_selected:
4411                 continue
4412             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4413             if label:
4414                 badge_labels.add(label.lower())
4415                 break
4416
4417         for badge_label in badge_labels:
4418             if badge_label == 'unlisted':
4419                 is_unlisted = True
4420             elif badge_label == 'private':
4421                 is_private = True
4422             elif badge_label == 'public':
4423                 is_unlisted = is_private = False
4424         return self._availability(is_private, False, False, False, is_unlisted)
4425
4426     @staticmethod
4427     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4428         sidebar_renderer = try_get(
4429             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4430         for item in sidebar_renderer:
4431             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4432             if renderer:
4433                 return renderer
4434
4435     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4436         """
4437         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4438         """
4439         browse_id = params = None
4440         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4441         if not renderer:
4442             return
4443         menu_renderer = try_get(
4444             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4445         for menu_item in menu_renderer:
4446             if not isinstance(menu_item, dict):
4447                 continue
4448             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4449             text = try_get(
4450                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4451             if not text or text.lower() != 'show unavailable videos':
4452                 continue
4453             browse_endpoint = try_get(
4454                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4455             browse_id = browse_endpoint.get('browseId')
4456             params = browse_endpoint.get('params')
4457             break
4458
4459         headers = self.generate_api_headers(
4460             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4461             visitor_data=self._extract_visitor_data(data, ytcfg))
4462         query = {
4463             'params': params or 'wgYCCAA=',
4464             'browseId': browse_id or 'VL%s' % item_id
4465         }
4466         return self._extract_response(
4467             item_id=item_id, headers=headers, query=query,
4468             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4469             note='Downloading API JSON with unavailable videos')
4470
4471     @functools.cached_property
4472     def skip_webpage(self):
4473         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4474
4475     def _extract_webpage(self, url, item_id, fatal=True):
4476         retries = self.get_param('extractor_retries', 3)
4477         count = -1
4478         webpage = data = last_error = None
4479         while count < retries:
4480             count += 1
4481             # Sometimes youtube returns a webpage with incomplete ytInitialData
4482             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4483             if last_error:
4484                 self.report_warning('%s. Retrying ...' % last_error)
4485             try:
4486                 webpage = self._download_webpage(
4487                     url, item_id,
4488                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4489                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4490             except ExtractorError as e:
4491                 if isinstance(e.cause, network_exceptions):
4492                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4493                         last_error = error_to_compat_str(e.cause or e.msg)
4494                         if count < retries:
4495                             continue
4496                 if fatal:
4497                     raise
4498                 self.report_warning(error_to_compat_str(e))
4499                 break
4500             else:
4501                 try:
4502                     self._extract_and_report_alerts(data)
4503                 except ExtractorError as e:
4504                     if fatal:
4505                         raise
4506                     self.report_warning(error_to_compat_str(e))
4507                     break
4508
4509                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4510                     break
4511
4512                 last_error = 'Incomplete yt initial data received'
4513                 if count >= retries:
4514                     if fatal:
4515                         raise ExtractorError(last_error)
4516                     self.report_warning(last_error)
4517                     break
4518
4519         return webpage, data
4520
4521     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4522         """Use if failed to extract ytcfg (and data) from initial webpage"""
4523         if not ytcfg and self.is_authenticated:
4524             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4525             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4526                 raise ExtractorError(
4527                     f'{msg}. If you are not downloading private content, or '
4528                     'your cookies are only for the first account and channel,'
4529                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4530                     expected=True)
4531             self.report_warning(msg, only_once=True)
4532
4533     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4534         data = None
4535         if not self.skip_webpage:
4536             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4537             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4538             # Reject webpage data if redirected to home page without explicitly requesting
4539             selected_tab = self._extract_selected_tab(traverse_obj(
4540                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4541             if (url != 'https://www.youtube.com/feed/recommended'
4542                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4543                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4544                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4545                 if fatal:
4546                     raise ExtractorError(msg, expected=True)
4547                 self.report_warning(msg, only_once=True)
4548         if not data:
4549             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4550             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4551         return data, ytcfg
4552
4553     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4554         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4555         resolve_response = self._extract_response(
4556             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4557             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4558         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4559         for ep_key, ep in endpoints.items():
4560             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4561             if params:
4562                 return self._extract_response(
4563                     item_id=item_id, query=params, ep=ep, headers=headers,
4564                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4565                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4566         err_note = 'Failed to resolve url (does the playlist exist?)'
4567         if fatal:
4568             raise ExtractorError(err_note, expected=True)
4569         self.report_warning(err_note, item_id)
4570
4571     _SEARCH_PARAMS = None
4572
4573     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4574         data = {'query': query}
4575         if params is NO_DEFAULT:
4576             params = self._SEARCH_PARAMS
4577         if params:
4578             data['params'] = params
4579
4580         content_keys = (
4581             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4582             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4583             # ytmusic search
4584             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4585             ('continuationContents', ),
4586         )
4587         display_id = f'query "{query}"'
4588         check_get_keys = tuple({keys[0] for keys in content_keys})
4589         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4590         self._report_playlist_authcheck(ytcfg, fatal=False)
4591
4592         continuation_list = [None]
4593         search = None
4594         for page_num in itertools.count(1):
4595             data.update(continuation_list[0] or {})
4596             headers = self.generate_api_headers(
4597                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4598             search = self._extract_response(
4599                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4600                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4601             slr_contents = traverse_obj(search, *content_keys)
4602             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4603             if not continuation_list[0]:
4604                 break
4605
4606
4607 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4608     IE_DESC = 'YouTube Tabs'
4609     _VALID_URL = r'''(?x:
4610         https?://
4611             (?:\w+\.)?
4612             (?:
4613                 youtube(?:kids)?\.com|
4614                 %(invidious)s
4615             )/
4616             (?:
4617                 (?P<channel_type>channel|c|user|browse)/|
4618                 (?P<not_channel>
4619                     feed/|hashtag/|
4620                     (?:playlist|watch)\?.*?\blist=
4621                 )|
4622                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4623             )
4624             (?P<id>[^/?\#&]+)
4625     )''' % {
4626         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4627         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4628     }
4629     IE_NAME = 'youtube:tab'
4630
4631     _TESTS = [{
4632         'note': 'playlists, multipage',
4633         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4634         'playlist_mincount': 94,
4635         'info_dict': {
4636             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4637             'title': 'Igor Kleiner - Playlists',
4638             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4639             'uploader': 'Igor Kleiner',
4640             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4641             'channel': 'Igor Kleiner',
4642             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4643             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4644             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4645             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4646             'channel_follower_count': int
4647         },
4648     }, {
4649         'note': 'playlists, multipage, different order',
4650         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4651         'playlist_mincount': 94,
4652         'info_dict': {
4653             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4654             'title': 'Igor Kleiner - Playlists',
4655             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4656             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4657             'uploader': 'Igor Kleiner',
4658             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4659             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4660             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4661             'channel': 'Igor Kleiner',
4662             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4663             'channel_follower_count': int
4664         },
4665     }, {
4666         'note': 'playlists, series',
4667         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4668         'playlist_mincount': 5,
4669         'info_dict': {
4670             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4671             'title': '3Blue1Brown - Playlists',
4672             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4673             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4674             'uploader': '3Blue1Brown',
4675             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4676             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4677             'channel': '3Blue1Brown',
4678             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4679             'tags': ['Mathematics'],
4680             'channel_follower_count': int
4681         },
4682     }, {
4683         'note': 'playlists, singlepage',
4684         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4685         'playlist_mincount': 4,
4686         'info_dict': {
4687             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4688             'title': 'ThirstForScience - Playlists',
4689             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4690             'uploader': 'ThirstForScience',
4691             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4692             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4693             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4694             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4695             'tags': 'count:13',
4696             'channel': 'ThirstForScience',
4697             'channel_follower_count': int
4698         }
4699     }, {
4700         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4701         'only_matching': True,
4702     }, {
4703         'note': 'basic, single video playlist',
4704         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4705         'info_dict': {
4706             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4707             'uploader': 'Sergey M.',
4708             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4709             'title': 'youtube-dl public playlist',
4710             'description': '',
4711             'tags': [],
4712             'view_count': int,
4713             'modified_date': '20201130',
4714             'channel': 'Sergey M.',
4715             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4716             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4717             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4718         },
4719         'playlist_count': 1,
4720     }, {
4721         'note': 'empty playlist',
4722         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4723         'info_dict': {
4724             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4725             'uploader': 'Sergey M.',
4726             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4727             'title': 'youtube-dl empty playlist',
4728             'tags': [],
4729             'channel': 'Sergey M.',
4730             'description': '',
4731             'modified_date': '20160902',
4732             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4733             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4734             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4735         },
4736         'playlist_count': 0,
4737     }, {
4738         'note': 'Home tab',
4739         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4740         'info_dict': {
4741             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4742             'title': 'lex will - Home',
4743             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4744             'uploader': 'lex will',
4745             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4746             'channel': 'lex will',
4747             'tags': ['bible', 'history', 'prophesy'],
4748             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4749             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4750             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4751             'channel_follower_count': int
4752         },
4753         'playlist_mincount': 2,
4754     }, {
4755         'note': 'Videos tab',
4756         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4757         'info_dict': {
4758             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4759             'title': 'lex will - Videos',
4760             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4761             'uploader': 'lex will',
4762             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4763             'tags': ['bible', 'history', 'prophesy'],
4764             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4765             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4766             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4767             'channel': 'lex will',
4768             'channel_follower_count': int
4769         },
4770         'playlist_mincount': 975,
4771     }, {
4772         'note': 'Videos tab, sorted by popular',
4773         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4774         'info_dict': {
4775             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4776             'title': 'lex will - Videos',
4777             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4778             'uploader': 'lex will',
4779             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4780             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4781             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4782             'channel': 'lex will',
4783             'tags': ['bible', 'history', 'prophesy'],
4784             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4785             'channel_follower_count': int
4786         },
4787         'playlist_mincount': 199,
4788     }, {
4789         'note': 'Playlists tab',
4790         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4791         'info_dict': {
4792             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4793             'title': 'lex will - Playlists',
4794             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4795             'uploader': 'lex will',
4796             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4797             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4798             'channel': 'lex will',
4799             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4800             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4801             'tags': ['bible', 'history', 'prophesy'],
4802             'channel_follower_count': int
4803         },
4804         'playlist_mincount': 17,
4805     }, {
4806         'note': 'Community tab',
4807         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4808         'info_dict': {
4809             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4810             'title': 'lex will - Community',
4811             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4812             'uploader': 'lex will',
4813             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4814             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4815             'channel': 'lex will',
4816             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4817             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4818             'tags': ['bible', 'history', 'prophesy'],
4819             'channel_follower_count': int
4820         },
4821         'playlist_mincount': 18,
4822     }, {
4823         'note': 'Channels tab',
4824         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4825         'info_dict': {
4826             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4827             'title': 'lex will - Channels',
4828             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4829             'uploader': 'lex will',
4830             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4831             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4832             'channel': 'lex will',
4833             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4834             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4835             'tags': ['bible', 'history', 'prophesy'],
4836             'channel_follower_count': int
4837         },
4838         'playlist_mincount': 12,
4839     }, {
4840         'note': 'Search tab',
4841         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4842         'playlist_mincount': 40,
4843         'info_dict': {
4844             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4845             'title': '3Blue1Brown - Search - linear algebra',
4846             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4847             'uploader': '3Blue1Brown',
4848             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4849             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4850             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4851             'tags': ['Mathematics'],
4852             'channel': '3Blue1Brown',
4853             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4854             'channel_follower_count': int
4855         },
4856     }, {
4857         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4858         'only_matching': True,
4859     }, {
4860         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4861         'only_matching': True,
4862     }, {
4863         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4864         'only_matching': True,
4865     }, {
4866         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4867         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4868         'info_dict': {
4869             'title': '29C3: Not my department',
4870             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4871             'uploader': 'Christiaan008',
4872             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4873             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4874             'tags': [],
4875             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4876             'view_count': int,
4877             'modified_date': '20150605',
4878             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4879             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4880             'channel': 'Christiaan008',
4881         },
4882         'playlist_count': 96,
4883     }, {
4884         'note': 'Large playlist',
4885         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4886         'info_dict': {
4887             'title': 'Uploads from Cauchemar',
4888             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4889             'uploader': 'Cauchemar',
4890             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4891             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4892             'tags': [],
4893             'modified_date': r're:\d{8}',
4894             'channel': 'Cauchemar',
4895             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4896             'view_count': int,
4897             'description': '',
4898             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4899         },
4900         'playlist_mincount': 1123,
4901         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4902     }, {
4903         'note': 'even larger playlist, 8832 videos',
4904         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4905         'only_matching': True,
4906     }, {
4907         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4908         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4909         'info_dict': {
4910             'title': 'Uploads from Interstellar Movie',
4911             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4912             'uploader': 'Interstellar Movie',
4913             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4914             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4915             'tags': [],
4916             'view_count': int,
4917             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4918             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4919             'channel': 'Interstellar Movie',
4920             'description': '',
4921             'modified_date': r're:\d{8}',
4922         },
4923         'playlist_mincount': 21,
4924     }, {
4925         'note': 'Playlist with "show unavailable videos" button',
4926         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4927         'info_dict': {
4928             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4929             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4930             'uploader': 'Phim Siêu Nhân Nhật Bản',
4931             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4932             'view_count': int,
4933             'channel': 'Phim Siêu Nhân Nhật Bản',
4934             'tags': [],
4935             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4936             'description': '',
4937             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4938             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4939             'modified_date': r're:\d{8}',
4940         },
4941         'playlist_mincount': 200,
4942         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4943     }, {
4944         'note': 'Playlist with unavailable videos in page 7',
4945         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4946         'info_dict': {
4947             'title': 'Uploads from BlankTV',
4948             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4949             'uploader': 'BlankTV',
4950             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4951             'channel': 'BlankTV',
4952             'channel_url': 'https://www.youtube.com/c/blanktv',
4953             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4954             'view_count': int,
4955             'tags': [],
4956             'uploader_url': 'https://www.youtube.com/c/blanktv',
4957             'modified_date': r're:\d{8}',
4958             'description': '',
4959         },
4960         'playlist_mincount': 1000,
4961         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4962     }, {
4963         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4964         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4965         'info_dict': {
4966             'title': 'Data Analysis with Dr Mike Pound',
4967             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4968             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4969             'uploader': 'Computerphile',
4970             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4971             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4972             'tags': [],
4973             'view_count': int,
4974             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4975             'channel_url': 'https://www.youtube.com/user/Computerphile',
4976             'channel': 'Computerphile',
4977         },
4978         'playlist_mincount': 11,
4979     }, {
4980         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4981         'only_matching': True,
4982     }, {
4983         'note': 'Playlist URL that does not actually serve a playlist',
4984         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4985         'info_dict': {
4986             'id': 'FqZTN594JQw',
4987             'ext': 'webm',
4988             'title': "Smiley's People 01 detective, Adventure Series, Action",
4989             'uploader': 'STREEM',
4990             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4991             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4992             'upload_date': '20150526',
4993             'license': 'Standard YouTube License',
4994             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4995             'categories': ['People & Blogs'],
4996             'tags': list,
4997             'view_count': int,
4998             'like_count': int,
4999         },
5000         'params': {
5001             'skip_download': True,
5002         },
5003         'skip': 'This video is not available.',
5004         'add_ie': [YoutubeIE.ie_key()],
5005     }, {
5006         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5007         'only_matching': True,
5008     }, {
5009         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5010         'only_matching': True,
5011     }, {
5012         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5013         'info_dict': {
5014             'id': 'GgL890LIznQ',  # This will keep changing
5015             'ext': 'mp4',
5016             'title': str,
5017             'uploader': 'Sky News',
5018             'uploader_id': 'skynews',
5019             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5020             'upload_date': r're:\d{8}',
5021             'description': str,
5022             'categories': ['News & Politics'],
5023             'tags': list,
5024             'like_count': int,
5025             'release_timestamp': 1642502819,
5026             'channel': 'Sky News',
5027             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5028             'age_limit': 0,
5029             'view_count': int,
5030             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5031             'playable_in_embed': True,
5032             'release_date': '20220118',
5033             'availability': 'public',
5034             'live_status': 'is_live',
5035             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5036             'channel_follower_count': int
5037         },
5038         'params': {
5039             'skip_download': True,
5040         },
5041         'expected_warnings': ['Ignoring subtitle tracks found in '],
5042     }, {
5043         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5044         'info_dict': {
5045             'id': 'a48o2S1cPoo',
5046             'ext': 'mp4',
5047             'title': 'The Young Turks - Live Main Show',
5048             'uploader': 'The Young Turks',
5049             'uploader_id': 'TheYoungTurks',
5050             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5051             'upload_date': '20150715',
5052             'license': 'Standard YouTube License',
5053             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5054             'categories': ['News & Politics'],
5055             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5056             'like_count': int,
5057         },
5058         'params': {
5059             'skip_download': True,
5060         },
5061         'only_matching': True,
5062     }, {
5063         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5064         'only_matching': True,
5065     }, {
5066         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5067         'only_matching': True,
5068     }, {
5069         'note': 'A channel that is not live. Should raise error',
5070         'url': 'https://www.youtube.com/user/numberphile/live',
5071         'only_matching': True,
5072     }, {
5073         'url': 'https://www.youtube.com/feed/trending',
5074         'only_matching': True,
5075     }, {
5076         'url': 'https://www.youtube.com/feed/library',
5077         'only_matching': True,
5078     }, {
5079         'url': 'https://www.youtube.com/feed/history',
5080         'only_matching': True,
5081     }, {
5082         'url': 'https://www.youtube.com/feed/subscriptions',
5083         'only_matching': True,
5084     }, {
5085         'url': 'https://www.youtube.com/feed/watch_later',
5086         'only_matching': True,
5087     }, {
5088         'note': 'Recommended - redirects to home page.',
5089         'url': 'https://www.youtube.com/feed/recommended',
5090         'only_matching': True,
5091     }, {
5092         'note': 'inline playlist with not always working continuations',
5093         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5094         'only_matching': True,
5095     }, {
5096         'url': 'https://www.youtube.com/course',
5097         'only_matching': True,
5098     }, {
5099         'url': 'https://www.youtube.com/zsecurity',
5100         'only_matching': True,
5101     }, {
5102         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5103         'only_matching': True,
5104     }, {
5105         'url': 'https://www.youtube.com/TheYoungTurks/live',
5106         'only_matching': True,
5107     }, {
5108         'url': 'https://www.youtube.com/hashtag/cctv9',
5109         'info_dict': {
5110             'id': 'cctv9',
5111             'title': '#cctv9',
5112             'tags': [],
5113         },
5114         'playlist_mincount': 350,
5115     }, {
5116         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5117         'only_matching': True,
5118     }, {
5119         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5120         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5121         'only_matching': True
5122     }, {
5123         'note': '/browse/ should redirect to /channel/',
5124         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5125         'only_matching': True
5126     }, {
5127         'note': 'VLPL, should redirect to playlist?list=PL...',
5128         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5129         'info_dict': {
5130             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5131             'uploader': 'NoCopyrightSounds',
5132             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5133             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5134             'title': 'NCS Releases',
5135             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5136             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5137             'modified_date': r're:\d{8}',
5138             'view_count': int,
5139             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5140             'tags': [],
5141             'channel': 'NoCopyrightSounds',
5142         },
5143         'playlist_mincount': 166,
5144         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5145     }, {
5146         'note': 'Topic, should redirect to playlist?list=UU...',
5147         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5148         'info_dict': {
5149             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5150             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5151             'title': 'Uploads from Royalty Free Music - Topic',
5152             'uploader': 'Royalty Free Music - Topic',
5153             'tags': [],
5154             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5155             'channel': 'Royalty Free Music - Topic',
5156             'view_count': int,
5157             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5158             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5159             'modified_date': r're:\d{8}',
5160             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5161             'description': '',
5162         },
5163         'expected_warnings': [
5164             'The URL does not have a videos tab',
5165             r'[Uu]navailable videos (are|will be) hidden',
5166         ],
5167         'playlist_mincount': 101,
5168     }, {
5169         'note': 'Topic without a UU playlist',
5170         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5171         'info_dict': {
5172             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5173             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5174             'tags': [],
5175         },
5176         'expected_warnings': [
5177             'the playlist redirect gave error',
5178         ],
5179         'playlist_mincount': 9,
5180     }, {
5181         'note': 'Youtube music Album',
5182         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5183         'info_dict': {
5184             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5185             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5186             'tags': [],
5187             'view_count': int,
5188             'description': '',
5189             'availability': 'unlisted',
5190             'modified_date': r're:\d{8}',
5191         },
5192         'playlist_count': 50,
5193     }, {
5194         'note': 'unlisted single video playlist',
5195         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5196         'info_dict': {
5197             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5198             'uploader': 'colethedj',
5199             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5200             'title': 'yt-dlp unlisted playlist test',
5201             'availability': 'unlisted',
5202             'tags': [],
5203             'modified_date': '20211208',
5204             'channel': 'colethedj',
5205             'view_count': int,
5206             'description': '',
5207             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5208             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5209             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5210         },
5211         'playlist_count': 1,
5212     }, {
5213         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5214         'url': 'https://www.youtube.com/feed/recommended',
5215         'info_dict': {
5216             'id': 'recommended',
5217             'title': 'recommended',
5218             'tags': [],
5219         },
5220         'playlist_mincount': 50,
5221         'params': {
5222             'skip_download': True,
5223             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5224         },
5225     }, {
5226         'note': 'API Fallback: /videos tab, sorted by oldest first',
5227         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5228         'info_dict': {
5229             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5230             'title': 'Cody\'sLab - Videos',
5231             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5232             'uploader': 'Cody\'sLab',
5233             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5234             'channel': 'Cody\'sLab',
5235             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5236             'tags': [],
5237             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5238             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5239             'channel_follower_count': int
5240         },
5241         'playlist_mincount': 650,
5242         'params': {
5243             'skip_download': True,
5244             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5245         },
5246     }, {
5247         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5248         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5249         'info_dict': {
5250             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5251             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5252             'title': 'Uploads from Royalty Free Music - Topic',
5253             'uploader': 'Royalty Free Music - Topic',
5254             'modified_date': r're:\d{8}',
5255             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5256             'description': '',
5257             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5258             'tags': [],
5259             'channel': 'Royalty Free Music - Topic',
5260             'view_count': int,
5261             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5262         },
5263         'expected_warnings': [
5264             'does not have a videos tab',
5265             r'[Uu]navailable videos (are|will be) hidden',
5266         ],
5267         'playlist_mincount': 101,
5268         'params': {
5269             'skip_download': True,
5270             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5271         },
5272     }, {
5273         'note': 'non-standard redirect to regional channel',
5274         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5275         'only_matching': True
5276     }, {
5277         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5278         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5279         'info_dict': {
5280             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5281             'modified_date': '20220407',
5282             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5283             'tags': [],
5284             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5285             'uploader': 'pukkandan',
5286             'availability': 'unlisted',
5287             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5288             'channel': 'pukkandan',
5289             'description': 'Test for collaborative playlist',
5290             'title': 'yt-dlp test - collaborative playlist',
5291             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5292         },
5293         'playlist_mincount': 2
5294     }]
5295
5296     @classmethod
5297     def suitable(cls, url):
5298         return False if YoutubeIE.suitable(url) else super().suitable(url)
5299
5300     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5301
5302     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5303     def _real_extract(self, url, smuggled_data):
5304         item_id = self._match_id(url)
5305         url = compat_urlparse.urlunparse(
5306             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
5307         compat_opts = self.get_param('compat_opts', [])
5308
5309         def get_mobj(url):
5310             mobj = self._URL_RE.match(url).groupdict()
5311             mobj.update((k, '') for k, v in mobj.items() if v is None)
5312             return mobj
5313
5314         mobj, redirect_warning = get_mobj(url), None
5315         # Youtube returns incomplete data if tabname is not lower case
5316         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5317         if is_channel:
5318             if smuggled_data.get('is_music_url'):
5319                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5320                     item_id = item_id[2:]
5321                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5322                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5323                     mdata = self._extract_tab_endpoint(
5324                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5325                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5326                                         get_all=False, expected_type=compat_str)
5327                     if not murl:
5328                         raise ExtractorError('Failed to resolve album to playlist')
5329                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5330                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5331                     pre = f'https://www.youtube.com/channel/{item_id}'
5332
5333         original_tab_name = tab
5334         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5335             # Home URLs should redirect to /videos/
5336             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5337                                 'To download only the videos in the home page, add a "/featured" to the URL')
5338             tab = '/videos'
5339
5340         url = ''.join((pre, tab, post))
5341         mobj = get_mobj(url)
5342
5343         # Handle both video/playlist URLs
5344         qs = parse_qs(url)
5345         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5346
5347         if not video_id and mobj['not_channel'].startswith('watch'):
5348             if not playlist_id:
5349                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5350                 raise ExtractorError('Unable to recognize tab page')
5351             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5352             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5353             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5354             mobj = get_mobj(url)
5355
5356         if video_id and playlist_id:
5357             if self.get_param('noplaylist'):
5358                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5359                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5360                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5361             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5362
5363         data, ytcfg = self._extract_data(url, item_id)
5364
5365         # YouTube may provide a non-standard redirect to the regional channel
5366         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5367         redirect_url = traverse_obj(
5368             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5369         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5370             redirect_url = ''.join((
5371                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5372             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5373             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5374
5375         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5376         if tabs:
5377             selected_tab = self._extract_selected_tab(tabs)
5378             selected_tab_name = selected_tab.get('title', '').lower()
5379             if selected_tab_name == 'home':
5380                 selected_tab_name = 'featured'
5381             requested_tab_name = mobj['tab'][1:]
5382             if 'no-youtube-channel-redirect' not in compat_opts:
5383                 if requested_tab_name == 'live':
5384                     # Live tab should have redirected to the video
5385                     raise ExtractorError('The channel is not currently live', expected=True)
5386                 if requested_tab_name not in ('', selected_tab_name):
5387                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5388                     if not original_tab_name:
5389                         if item_id[:2] == 'UC':
5390                             # Topic channels don't have /videos. Use the equivalent playlist instead
5391                             pl_id = f'UU{item_id[2:]}'
5392                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5393                             try:
5394                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5395                             except ExtractorError:
5396                                 redirect_warning += ' and the playlist redirect gave error'
5397                             else:
5398                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5399                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5400                         if selected_tab_name and selected_tab_name != requested_tab_name:
5401                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5402                     else:
5403                         raise ExtractorError(redirect_warning, expected=True)
5404
5405         if redirect_warning:
5406             self.to_screen(redirect_warning)
5407         self.write_debug(f'Final URL: {url}')
5408
5409         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5410         if 'no-youtube-unavailable-videos' not in compat_opts:
5411             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5412         self._extract_and_report_alerts(data, only_once=True)
5413         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5414         if tabs:
5415             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5416
5417         playlist = traverse_obj(
5418             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5419         if playlist:
5420             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5421
5422         video_id = traverse_obj(
5423             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5424         if video_id:
5425             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5426                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5427             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5428                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5429
5430         raise ExtractorError('Unable to recognize tab page')
5431
5432
5433 class YoutubePlaylistIE(InfoExtractor):
5434     IE_DESC = 'YouTube playlists'
5435     _VALID_URL = r'''(?x)(?:
5436                         (?:https?://)?
5437                         (?:\w+\.)?
5438                         (?:
5439                             (?:
5440                                 youtube(?:kids)?\.com|
5441                                 %(invidious)s
5442                             )
5443                             /.*?\?.*?\blist=
5444                         )?
5445                         (?P<id>%(playlist_id)s)
5446                      )''' % {
5447         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5448         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5449     }
5450     IE_NAME = 'youtube:playlist'
5451     _TESTS = [{
5452         'note': 'issue #673',
5453         'url': 'PLBB231211A4F62143',
5454         'info_dict': {
5455             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5456             'id': 'PLBB231211A4F62143',
5457             'uploader': 'Wickman',
5458             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5459             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5460             'view_count': int,
5461             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5462             'modified_date': r're:\d{8}',
5463             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5464             'channel': 'Wickman',
5465             'tags': [],
5466             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5467         },
5468         'playlist_mincount': 29,
5469     }, {
5470         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5471         'info_dict': {
5472             'title': 'YDL_safe_search',
5473             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5474         },
5475         'playlist_count': 2,
5476         'skip': 'This playlist is private',
5477     }, {
5478         'note': 'embedded',
5479         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5480         'playlist_count': 4,
5481         'info_dict': {
5482             'title': 'JODA15',
5483             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5484             'uploader': 'milan',
5485             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5486             'description': '',
5487             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5488             'tags': [],
5489             'modified_date': '20140919',
5490             'view_count': int,
5491             'channel': 'milan',
5492             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5493             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5494         },
5495         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5496     }, {
5497         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5498         'playlist_mincount': 654,
5499         'info_dict': {
5500             'title': '2018 Chinese New Singles (11/6 updated)',
5501             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5502             'uploader': 'LBK',
5503             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5504             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5505             'channel': 'LBK',
5506             'view_count': int,
5507             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5508             'tags': [],
5509             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5510             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5511             'modified_date': r're:\d{8}',
5512         },
5513         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5514     }, {
5515         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5516         'only_matching': True,
5517     }, {
5518         # music album playlist
5519         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5520         'only_matching': True,
5521     }]
5522
5523     @classmethod
5524     def suitable(cls, url):
5525         if YoutubeTabIE.suitable(url):
5526             return False
5527         from ..utils import parse_qs
5528         qs = parse_qs(url)
5529         if qs.get('v', [None])[0]:
5530             return False
5531         return super().suitable(url)
5532
5533     def _real_extract(self, url):
5534         playlist_id = self._match_id(url)
5535         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5536         url = update_url_query(
5537             'https://www.youtube.com/playlist',
5538             parse_qs(url) or {'list': playlist_id})
5539         if is_music_url:
5540             url = smuggle_url(url, {'is_music_url': True})
5541         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5542
5543
5544 class YoutubeYtBeIE(InfoExtractor):
5545     IE_DESC = 'youtu.be'
5546     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5547     _TESTS = [{
5548         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5549         'info_dict': {
5550             'id': 'yeWKywCrFtk',
5551             'ext': 'mp4',
5552             'title': 'Small Scale Baler and Braiding Rugs',
5553             'uploader': 'Backus-Page House Museum',
5554             'uploader_id': 'backuspagemuseum',
5555             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5556             'upload_date': '20161008',
5557             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5558             'categories': ['Nonprofits & Activism'],
5559             'tags': list,
5560             'like_count': int,
5561             'age_limit': 0,
5562             'playable_in_embed': True,
5563             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5564             'channel': 'Backus-Page House Museum',
5565             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5566             'live_status': 'not_live',
5567             'view_count': int,
5568             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5569             'availability': 'public',
5570             'duration': 59,
5571         },
5572         'params': {
5573             'noplaylist': True,
5574             'skip_download': True,
5575         },
5576     }, {
5577         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5578         'only_matching': True,
5579     }]
5580
5581     def _real_extract(self, url):
5582         mobj = self._match_valid_url(url)
5583         video_id = mobj.group('id')
5584         playlist_id = mobj.group('playlist_id')
5585         return self.url_result(
5586             update_url_query('https://www.youtube.com/watch', {
5587                 'v': video_id,
5588                 'list': playlist_id,
5589                 'feature': 'youtu.be',
5590             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5591
5592
5593 class YoutubeLivestreamEmbedIE(InfoExtractor):
5594     IE_DESC = 'YouTube livestream embeds'
5595     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5596     _TESTS = [{
5597         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5598         'only_matching': True,
5599     }]
5600
5601     def _real_extract(self, url):
5602         channel_id = self._match_id(url)
5603         return self.url_result(
5604             f'https://www.youtube.com/channel/{channel_id}/live',
5605             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5606
5607
5608 class YoutubeYtUserIE(InfoExtractor):
5609     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5610     IE_NAME = 'youtube:user'
5611     _VALID_URL = r'ytuser:(?P<id>.+)'
5612     _TESTS = [{
5613         'url': 'ytuser:phihag',
5614         'only_matching': True,
5615     }]
5616
5617     def _real_extract(self, url):
5618         user_id = self._match_id(url)
5619         return self.url_result(
5620             'https://www.youtube.com/user/%s/videos' % user_id,
5621             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5622
5623
5624 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5625     IE_NAME = 'youtube:favorites'
5626     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5627     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5628     _LOGIN_REQUIRED = True
5629     _TESTS = [{
5630         'url': ':ytfav',
5631         'only_matching': True,
5632     }, {
5633         'url': ':ytfavorites',
5634         'only_matching': True,
5635     }]
5636
5637     def _real_extract(self, url):
5638         return self.url_result(
5639             'https://www.youtube.com/playlist?list=LL',
5640             ie=YoutubeTabIE.ie_key())
5641
5642
5643 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5644     IE_NAME = 'youtube:notif'
5645     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5646     _VALID_URL = r':ytnotif(?:ication)?s?'
5647     _LOGIN_REQUIRED = True
5648     _TESTS = [{
5649         'url': ':ytnotif',
5650         'only_matching': True,
5651     }, {
5652         'url': ':ytnotifications',
5653         'only_matching': True,
5654     }]
5655
5656     def _extract_notification_menu(self, response, continuation_list):
5657         notification_list = traverse_obj(
5658             response,
5659             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5660             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5661             expected_type=list) or []
5662         continuation_list[0] = None
5663         for item in notification_list:
5664             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5665             if entry:
5666                 yield entry
5667             continuation = item.get('continuationItemRenderer')
5668             if continuation:
5669                 continuation_list[0] = continuation
5670
5671     def _extract_notification_renderer(self, notification):
5672         video_id = traverse_obj(
5673             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5674         url = f'https://www.youtube.com/watch?v={video_id}'
5675         channel_id = None
5676         if not video_id:
5677             browse_ep = traverse_obj(
5678                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5679             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5680             post_id = self._search_regex(
5681                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5682                 'post id', default=None)
5683             if not channel_id or not post_id:
5684                 return
5685             # The direct /post url redirects to this in the browser
5686             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5687
5688         channel = traverse_obj(
5689             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5690             expected_type=str)
5691         notification_title = self._get_text(notification, 'shortMessage')
5692         if notification_title:
5693             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
5694         # TODO: handle recommended videos
5695         title = self._search_regex(
5696             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
5697             'video title', default=None)
5698         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5699                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5700                        else None)
5701         return {
5702             '_type': 'url',
5703             'url': url,
5704             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5705             'video_id': video_id,
5706             'title': title,
5707             'channel_id': channel_id,
5708             'channel': channel,
5709             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5710             'upload_date': upload_date,
5711         }
5712
5713     def _notification_menu_entries(self, ytcfg):
5714         continuation_list = [None]
5715         response = None
5716         for page in itertools.count(1):
5717             ctoken = traverse_obj(
5718                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5719             response = self._extract_response(
5720                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5721                 ep='notification/get_notification_menu', check_get_keys='actions',
5722                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5723             yield from self._extract_notification_menu(response, continuation_list)
5724             if not continuation_list[0]:
5725                 break
5726
5727     def _real_extract(self, url):
5728         display_id = 'notifications'
5729         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5730         self._report_playlist_authcheck(ytcfg)
5731         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5732
5733
5734 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5735     IE_DESC = 'YouTube search'
5736     IE_NAME = 'youtube:search'
5737     _SEARCH_KEY = 'ytsearch'
5738     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5739     _TESTS = [{
5740         'url': 'ytsearch5:youtube-dl test video',
5741         'playlist_count': 5,
5742         'info_dict': {
5743             'id': 'youtube-dl test video',
5744             'title': 'youtube-dl test video',
5745         }
5746     }]
5747
5748
5749 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5750     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5751     _SEARCH_KEY = 'ytsearchdate'
5752     IE_DESC = 'YouTube search, newest videos first'
5753     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5754     _TESTS = [{
5755         'url': 'ytsearchdate5:youtube-dl test video',
5756         'playlist_count': 5,
5757         'info_dict': {
5758             'id': 'youtube-dl test video',
5759             'title': 'youtube-dl test video',
5760         }
5761     }]
5762
5763
5764 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5765     IE_DESC = 'YouTube search URLs with sorting and filter support'
5766     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5767     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5768     _TESTS = [{
5769         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5770         'playlist_mincount': 5,
5771         'info_dict': {
5772             'id': 'youtube-dl test video',
5773             'title': 'youtube-dl test video',
5774         }
5775     }, {
5776         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5777         'playlist_mincount': 5,
5778         'info_dict': {
5779             'id': 'python',
5780             'title': 'python',
5781         }
5782     }, {
5783         'url': 'https://www.youtube.com/results?search_query=%23cats',
5784         'playlist_mincount': 1,
5785         'info_dict': {
5786             'id': '#cats',
5787             'title': '#cats',
5788             'entries': [{
5789                 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5790                 'title': '#cats',
5791             }],
5792         },
5793     }, {
5794         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5795         'only_matching': True,
5796     }]
5797
5798     def _real_extract(self, url):
5799         qs = parse_qs(url)
5800         query = (qs.get('search_query') or qs.get('q'))[0]
5801         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5802
5803
5804 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5805     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5806     IE_NAME = 'youtube:music:search_url'
5807     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5808     _TESTS = [{
5809         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5810         'playlist_count': 16,
5811         'info_dict': {
5812             'id': 'royalty free music',
5813             'title': 'royalty free music',
5814         }
5815     }, {
5816         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5817         'playlist_mincount': 30,
5818         'info_dict': {
5819             'id': 'royalty free music - songs',
5820             'title': 'royalty free music - songs',
5821         },
5822         'params': {'extract_flat': 'in_playlist'}
5823     }, {
5824         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5825         'playlist_mincount': 30,
5826         'info_dict': {
5827             'id': 'royalty free music - community playlists',
5828             'title': 'royalty free music - community playlists',
5829         },
5830         'params': {'extract_flat': 'in_playlist'}
5831     }]
5832
5833     _SECTIONS = {
5834         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5835         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5836         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5837         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5838         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5839         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5840     }
5841
5842     def _real_extract(self, url):
5843         qs = parse_qs(url)
5844         query = (qs.get('search_query') or qs.get('q'))[0]
5845         params = qs.get('sp', (None,))[0]
5846         if params:
5847             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5848         else:
5849             section = compat_urllib_parse_unquote_plus((url.split('#') + [''])[1]).lower()
5850             params = self._SECTIONS.get(section)
5851             if not params:
5852                 section = None
5853         title = join_nonempty(query, section, delim=' - ')
5854         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5855
5856
5857 class YoutubeFeedsInfoExtractor(InfoExtractor):
5858     """
5859     Base class for feed extractors
5860     Subclasses must re-define the _FEED_NAME property.
5861     """
5862     _LOGIN_REQUIRED = True
5863     _FEED_NAME = 'feeds'
5864
5865     def _real_initialize(self):
5866         YoutubeBaseInfoExtractor._check_login_required(self)
5867
5868     @classproperty
5869     def IE_NAME(self):
5870         return f'youtube:{self._FEED_NAME}'
5871
5872     def _real_extract(self, url):
5873         return self.url_result(
5874             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5875
5876
5877 class YoutubeWatchLaterIE(InfoExtractor):
5878     IE_NAME = 'youtube:watchlater'
5879     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5880     _VALID_URL = r':ytwatchlater'
5881     _TESTS = [{
5882         'url': ':ytwatchlater',
5883         'only_matching': True,
5884     }]
5885
5886     def _real_extract(self, url):
5887         return self.url_result(
5888             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5889
5890
5891 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5892     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5893     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5894     _FEED_NAME = 'recommended'
5895     _LOGIN_REQUIRED = False
5896     _TESTS = [{
5897         'url': ':ytrec',
5898         'only_matching': True,
5899     }, {
5900         'url': ':ytrecommended',
5901         'only_matching': True,
5902     }, {
5903         'url': 'https://youtube.com',
5904         'only_matching': True,
5905     }]
5906
5907
5908 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5909     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5910     _VALID_URL = r':ytsub(?:scription)?s?'
5911     _FEED_NAME = 'subscriptions'
5912     _TESTS = [{
5913         'url': ':ytsubs',
5914         'only_matching': True,
5915     }, {
5916         'url': ':ytsubscriptions',
5917         'only_matching': True,
5918     }]
5919
5920
5921 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5922     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5923     _VALID_URL = r':ythis(?:tory)?'
5924     _FEED_NAME = 'history'
5925     _TESTS = [{
5926         'url': ':ythistory',
5927         'only_matching': True,
5928     }]
5929
5930
5931 class YoutubeStoriesIE(InfoExtractor):
5932     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5933     IE_NAME = 'youtube:stories'
5934     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5935     _TESTS = [{
5936         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5937         'only_matching': True,
5938     }]
5939
5940     def _real_extract(self, url):
5941         playlist_id = f'RLTD{self._match_id(url)}'
5942         return self.url_result(
5943             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5944             ie=YoutubeTabIE, video_id=playlist_id)
5945
5946
5947 class YoutubeTruncatedURLIE(InfoExtractor):
5948     IE_NAME = 'youtube:truncated_url'
5949     IE_DESC = False  # Do not list
5950     _VALID_URL = r'''(?x)
5951         (?:https?://)?
5952         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5953         (?:watch\?(?:
5954             feature=[a-z_]+|
5955             annotation_id=annotation_[^&]+|
5956             x-yt-cl=[0-9]+|
5957             hl=[^&]*|
5958             t=[0-9]+
5959         )?
5960         |
5961             attribution_link\?a=[^&]+
5962         )
5963         $
5964     '''
5965
5966     _TESTS = [{
5967         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5968         'only_matching': True,
5969     }, {
5970         'url': 'https://www.youtube.com/watch?',
5971         'only_matching': True,
5972     }, {
5973         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5974         'only_matching': True,
5975     }, {
5976         'url': 'https://www.youtube.com/watch?feature=foo',
5977         'only_matching': True,
5978     }, {
5979         'url': 'https://www.youtube.com/watch?hl=en-GB',
5980         'only_matching': True,
5981     }, {
5982         'url': 'https://www.youtube.com/watch?t=2372',
5983         'only_matching': True,
5984     }]
5985
5986     def _real_extract(self, url):
5987         raise ExtractorError(
5988             'Did you forget to quote the URL? Remember that & is a meta '
5989             'character in most shells, so you want to put the URL in quotes, '
5990             'like  youtube-dl '
5991             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5992             ' or simply  youtube-dl BaW_jenozKc  .',
5993             expected=True)
5994
5995
5996 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
5997     IE_NAME = 'youtube:clip'
5998     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
5999     _TESTS = [{
6000         # FIXME: Other metadata should be extracted from the clip, not from the base video
6001         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6002         'info_dict': {
6003             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6004             'ext': 'mp4',
6005             'section_start': 29.0,
6006             'section_end': 39.7,
6007             'duration': 10.7,
6008         }
6009     }]
6010
6011     def _real_extract(self, url):
6012         clip_id = self._match_id(url)
6013         _, data = self._extract_webpage(url, clip_id)
6014
6015         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6016         if not video_id:
6017             raise ExtractorError('Unable to find video ID')
6018
6019         clip_data = traverse_obj(data, (
6020             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6021             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6022             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6023             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6024
6025         return {
6026             '_type': 'url_transparent',
6027             'url': f'https://www.youtube.com/watch?v={video_id}',
6028             'ie_key': YoutubeIE.ie_key(),
6029             'id': clip_id,
6030             'section_start': int(clip_data['startTimeMs']) / 1000,
6031             'section_end': int(clip_data['endTimeMs']) / 1000,
6032         }
6033
6034
6035 class YoutubeTruncatedIDIE(InfoExtractor):
6036     IE_NAME = 'youtube:truncated_id'
6037     IE_DESC = False  # Do not list
6038     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6039
6040     _TESTS = [{
6041         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6042         'only_matching': True,
6043     }]
6044
6045     def _real_extract(self, url):
6046         video_id = self._match_id(url)
6047         raise ExtractorError(
6048             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6049             expected=True)