yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16 import urllib.error
  17 import urllib.parse
  18
  19 from .common import InfoExtractor, SearchInfoExtractor
  20 from ..compat import functools
  21 from ..jsinterp import JSInterpreter
  22 from ..utils import (
  23     NO_DEFAULT,
  24     ExtractorError,
  25     UserNotLive,
  26     bug_reports_message,
  27     classproperty,
  28     clean_html,
  29     datetime_from_str,
  30     dict_get,
  31     float_or_none,
  32     format_field,
  33     get_first,
  34     int_or_none,
  35     is_html,
  36     join_nonempty,
  37     js_to_json,
  38     mimetype2ext,
  39     network_exceptions,
  40     orderedSet,
  41     parse_codecs,
  42     parse_count,
  43     parse_duration,
  44     parse_iso8601,
  45     parse_qs,
  46     qualities,
  47     remove_start,
  48     smuggle_url,
  49     str_or_none,
  50     str_to_int,
  51     strftime_or_none,
  52     traverse_obj,
  53     try_get,
  54     unescapeHTML,
  55     unified_strdate,
  56     unified_timestamp,
  57     unsmuggle_url,
  58     update_url_query,
  59     url_or_none,
  60     urljoin,
  61     variadic,
  62 )
  63
  64 # any clients starting with _ cannot be explicitly requested by the user
  65 INNERTUBE_CLIENTS = {
  66     'web': {
  67         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  68         'INNERTUBE_CONTEXT': {
  69             'client': {
  70                 'clientName': 'WEB',
  71                 'clientVersion': '2.20220801.00.00',
  72             }
  73         },
  74         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  75     },
  76     'web_embedded': {
  77         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  78         'INNERTUBE_CONTEXT': {
  79             'client': {
  80                 'clientName': 'WEB_EMBEDDED_PLAYER',
  81                 'clientVersion': '1.20220731.00.00',
  82             },
  83         },
  84         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  85     },
  86     'web_music': {
  87         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  88         'INNERTUBE_HOST': 'music.youtube.com',
  89         'INNERTUBE_CONTEXT': {
  90             'client': {
  91                 'clientName': 'WEB_REMIX',
  92                 'clientVersion': '1.20220727.01.00',
  93             }
  94         },
  95         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
  96     },
  97     'web_creator': {
  98         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
  99         'INNERTUBE_CONTEXT': {
 100             'client': {
 101                 'clientName': 'WEB_CREATOR',
 102                 'clientVersion': '1.20220726.00.00',
 103             }
 104         },
 105         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 106     },
 107     'android': {
 108         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 109         'INNERTUBE_CONTEXT': {
 110             'client': {
 111                 'clientName': 'ANDROID',
 112                 'clientVersion': '17.28.34',
 113             }
 114         },
 115         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 116         'REQUIRE_JS_PLAYER': False
 117     },
 118     'android_embedded': {
 119         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 120         'INNERTUBE_CONTEXT': {
 121             'client': {
 122                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 123                 'clientVersion': '17.28.34',
 124             },
 125         },
 126         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 127         'REQUIRE_JS_PLAYER': False
 128     },
 129     'android_music': {
 130         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 131         'INNERTUBE_CONTEXT': {
 132             'client': {
 133                 'clientName': 'ANDROID_MUSIC',
 134                 'clientVersion': '5.16.51',
 135             }
 136         },
 137         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 138         'REQUIRE_JS_PLAYER': False
 139     },
 140     'android_creator': {
 141         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 142         'INNERTUBE_CONTEXT': {
 143             'client': {
 144                 'clientName': 'ANDROID_CREATOR',
 145                 'clientVersion': '22.28.100',
 146             },
 147         },
 148         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 149         'REQUIRE_JS_PLAYER': False
 150     },
 151     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 152     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 153     'ios': {
 154         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 155         'INNERTUBE_CONTEXT': {
 156             'client': {
 157                 'clientName': 'IOS',
 158                 'clientVersion': '17.30.1',
 159                 'deviceModel': 'iPhone14,3',
 160             }
 161         },
 162         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 163         'REQUIRE_JS_PLAYER': False
 164     },
 165     'ios_embedded': {
 166         'INNERTUBE_CONTEXT': {
 167             'client': {
 168                 'clientName': 'IOS_MESSAGES_EXTENSION',
 169                 'clientVersion': '17.30.1',
 170                 'deviceModel': 'iPhone14,3',
 171             },
 172         },
 173         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 174         'REQUIRE_JS_PLAYER': False
 175     },
 176     'ios_music': {
 177         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 178         'INNERTUBE_CONTEXT': {
 179             'client': {
 180                 'clientName': 'IOS_MUSIC',
 181                 'clientVersion': '5.18',
 182             },
 183         },
 184         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 185         'REQUIRE_JS_PLAYER': False
 186     },
 187     'ios_creator': {
 188         'INNERTUBE_CONTEXT': {
 189             'client': {
 190                 'clientName': 'IOS_CREATOR',
 191                 'clientVersion': '22.29.101',
 192             },
 193         },
 194         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 195         'REQUIRE_JS_PLAYER': False
 196     },
 197     # mweb has 'ultralow' formats
 198     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 199     'mweb': {
 200         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 201         'INNERTUBE_CONTEXT': {
 202             'client': {
 203                 'clientName': 'MWEB',
 204                 'clientVersion': '2.20220801.00.00',
 205             }
 206         },
 207         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 208     },
 209     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 210     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 211     'tv_embedded': {
 212         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 213         'INNERTUBE_CONTEXT': {
 214             'client': {
 215                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 216                 'clientVersion': '2.0',
 217             },
 218         },
 219         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 220     },
 221 }
 222
 223
 224 def _split_innertube_client(client_name):
 225     variant, *base = client_name.rsplit('.', 1)
 226     if base:
 227         return variant, base[0], variant
 228     base, *variant = client_name.split('_', 1)
 229     return client_name, base, variant[0] if variant else None
 230
 231
 232 def build_innertube_clients():
 233     THIRD_PARTY = {
 234         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 235     }
 236     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 237     priority = qualities(BASE_CLIENTS[::-1])
 238
 239     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 240         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 241         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 242         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 243         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 244
 245         _, base_client, variant = _split_innertube_client(client)
 246         ytcfg['priority'] = 10 * priority(base_client)
 247
 248         if not variant:
 249             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 250             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 251             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 252             embedscreen['priority'] -= 3
 253         elif variant == 'embedded':
 254             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 255             ytcfg['priority'] -= 2
 256         else:
 257             ytcfg['priority'] -= 3
 258
 259
 260 build_innertube_clients()
 261
 262
 263 class YoutubeBaseInfoExtractor(InfoExtractor):
 264     """Provide base functions for Youtube extractors"""
 265
 266     _RESERVED_NAMES = (
 267         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 268         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 269         r'browse|oembed|get_video_info|iframe_api|s/player|'
 270         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 271
 272     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 273
 274     # _NETRC_MACHINE = 'youtube'
 275
 276     # If True it will raise an error if no login info is provided
 277     _LOGIN_REQUIRED = False
 278
 279     _INVIDIOUS_SITES = (
 280         # invidious-redirect websites
 281         r'(?:www\.)?redirect\.invidious\.io',
 282         r'(?:(?:www|dev)\.)?invidio\.us',
 283         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 284         r'(?:www\.)?invidious\.pussthecat\.org',
 285         r'(?:www\.)?invidious\.zee\.li',
 286         r'(?:www\.)?invidious\.ethibox\.fr',
 287         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 288         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 289         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 290         # youtube-dl invidious instances list
 291         r'(?:(?:www|no)\.)?invidiou\.sh',
 292         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 293         r'(?:www\.)?invidious\.kabi\.tk',
 294         r'(?:www\.)?invidious\.mastodon\.host',
 295         r'(?:www\.)?invidious\.zapashcanon\.fr',
 296         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 297         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 298         r'(?:www\.)?invidious\.himiko\.cloud',
 299         r'(?:www\.)?invidious\.reallyancient\.tech',
 300         r'(?:www\.)?invidious\.tube',
 301         r'(?:www\.)?invidiou\.site',
 302         r'(?:www\.)?invidious\.site',
 303         r'(?:www\.)?invidious\.xyz',
 304         r'(?:www\.)?invidious\.nixnet\.xyz',
 305         r'(?:www\.)?invidious\.048596\.xyz',
 306         r'(?:www\.)?invidious\.drycat\.fr',
 307         r'(?:www\.)?inv\.skyn3t\.in',
 308         r'(?:www\.)?tube\.poal\.co',
 309         r'(?:www\.)?tube\.connect\.cafe',
 310         r'(?:www\.)?vid\.wxzm\.sx',
 311         r'(?:www\.)?vid\.mint\.lgbt',
 312         r'(?:www\.)?vid\.puffyan\.us',
 313         r'(?:www\.)?yewtu\.be',
 314         r'(?:www\.)?yt\.elukerio\.org',
 315         r'(?:www\.)?yt\.lelux\.fi',
 316         r'(?:www\.)?invidious\.ggc-project\.de',
 317         r'(?:www\.)?yt\.maisputain\.ovh',
 318         r'(?:www\.)?ytprivate\.com',
 319         r'(?:www\.)?invidious\.13ad\.de',
 320         r'(?:www\.)?invidious\.toot\.koeln',
 321         r'(?:www\.)?invidious\.fdn\.fr',
 322         r'(?:www\.)?watch\.nettohikari\.com',
 323         r'(?:www\.)?invidious\.namazso\.eu',
 324         r'(?:www\.)?invidious\.silkky\.cloud',
 325         r'(?:www\.)?invidious\.exonip\.de',
 326         r'(?:www\.)?invidious\.riverside\.rocks',
 327         r'(?:www\.)?invidious\.blamefran\.net',
 328         r'(?:www\.)?invidious\.moomoo\.de',
 329         r'(?:www\.)?ytb\.trom\.tf',
 330         r'(?:www\.)?yt\.cyberhost\.uk',
 331         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 332         r'(?:www\.)?qklhadlycap4cnod\.onion',
 333         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 334         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 335         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 336         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 337         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 338         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 339         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 340         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 341         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 342         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 343         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 344         r'(?:www\.)?piped\.kavin\.rocks',
 345         r'(?:www\.)?piped\.silkky\.cloud',
 346         r'(?:www\.)?piped\.tokhmi\.xyz',
 347         r'(?:www\.)?piped\.moomoo\.me',
 348         r'(?:www\.)?il\.ax',
 349         r'(?:www\.)?piped\.syncpundit\.com',
 350         r'(?:www\.)?piped\.mha\.fi',
 351         r'(?:www\.)?piped\.mint\.lgbt',
 352         r'(?:www\.)?piped\.privacy\.com\.de',
 353     )
 354
 355     def _initialize_consent(self):
 356         cookies = self._get_cookies('https://www.youtube.com/')
 357         if cookies.get('__Secure-3PSID'):
 358             return
 359         consent_id = None
 360         consent = cookies.get('CONSENT')
 361         if consent:
 362             if 'YES' in consent.value:
 363                 return
 364             consent_id = self._search_regex(
 365                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 366         if not consent_id:
 367             consent_id = random.randint(100, 999)
 368         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 369
 370     def _initialize_pref(self):
 371         cookies = self._get_cookies('https://www.youtube.com/')
 372         pref_cookie = cookies.get('PREF')
 373         pref = {}
 374         if pref_cookie:
 375             try:
 376                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 377             except ValueError:
 378                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 379         pref.update({'hl': 'en', 'tz': 'UTC'})
 380         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 381
 382     def _real_initialize(self):
 383         self._initialize_pref()
 384         self._initialize_consent()
 385         self._check_login_required()
 386
 387     def _check_login_required(self):
 388         if self._LOGIN_REQUIRED and not self._cookies_passed:
 389             self.raise_login_required('Login details are needed to download this content', method='cookies')
 390
 391     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 392     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 393
 394     def _get_default_ytcfg(self, client='web'):
 395         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 396
 397     def _get_innertube_host(self, client='web'):
 398         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 399
 400     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 401         # try_get but with fallback to default ytcfg client values when present
 402         _func = lambda y: try_get(y, getter, expected_type)
 403         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 404
 405     def _extract_client_name(self, ytcfg, default_client='web'):
 406         return self._ytcfg_get_safe(
 407             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 408                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 409
 410     def _extract_client_version(self, ytcfg, default_client='web'):
 411         return self._ytcfg_get_safe(
 412             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 413                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 414
 415     def _select_api_hostname(self, req_api_hostname, default_client=None):
 416         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 417                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 418
 419     def _extract_api_key(self, ytcfg=None, default_client='web'):
 420         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 421
 422     def _extract_context(self, ytcfg=None, default_client='web'):
 423         context = get_first(
 424             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 425         # Enforce language and tz for extraction
 426         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 427         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 428         return context
 429
 430     _SAPISID = None
 431
 432     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 433         time_now = round(time.time())
 434         if self._SAPISID is None:
 435             yt_cookies = self._get_cookies('https://www.youtube.com')
 436             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 437             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 438             sapisid_cookie = dict_get(
 439                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 440             if sapisid_cookie and sapisid_cookie.value:
 441                 self._SAPISID = sapisid_cookie.value
 442                 self.write_debug('Extracted SAPISID cookie')
 443                 # SAPISID cookie is required if not already present
 444                 if not yt_cookies.get('SAPISID'):
 445                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 446                     self._set_cookie(
 447                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 448             else:
 449                 self._SAPISID = False
 450         if not self._SAPISID:
 451             return None
 452         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 453         sapisidhash = hashlib.sha1(
 454             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 455         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 456
 457     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 458                   note='Downloading API JSON', errnote='Unable to download API page',
 459                   context=None, api_key=None, api_hostname=None, default_client='web'):
 460
 461         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 462         data.update(query)
 463         real_headers = self.generate_api_headers(default_client=default_client)
 464         real_headers.update({'content-type': 'application/json'})
 465         if headers:
 466             real_headers.update(headers)
 467         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 468                    or api_key or self._extract_api_key(default_client=default_client))
 469         return self._download_json(
 470             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 471             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 472             data=json.dumps(data).encode('utf8'), headers=real_headers,
 473             query={'key': api_key, 'prettyPrint': 'false'})
 474
 475     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 476         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 477
 478     @staticmethod
 479     def _extract_session_index(*data):
 480         """
 481         Index of current account in account list.
 482         See: https://github.com/yt-dlp/yt-dlp/pull/519
 483         """
 484         for ytcfg in data:
 485             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 486             if session_index is not None:
 487                 return session_index
 488
 489     # Deprecated?
 490     def _extract_identity_token(self, ytcfg=None, webpage=None):
 491         if ytcfg:
 492             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 493             if token:
 494                 return token
 495         if webpage:
 496             return self._search_regex(
 497                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 498                 'identity token', default=None, fatal=False)
 499
 500     @staticmethod
 501     def _extract_account_syncid(*args):
 502         """
 503         Extract syncId required to download private playlists of secondary channels
 504         @params response and/or ytcfg
 505         """
 506         for data in args:
 507             # ytcfg includes channel_syncid if on secondary channel
 508             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 509             if delegated_sid:
 510                 return delegated_sid
 511             sync_ids = (try_get(
 512                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 513                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 514             if len(sync_ids) >= 2 and sync_ids[1]:
 515                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 516                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 517                 return sync_ids[0]
 518
 519     @staticmethod
 520     def _extract_visitor_data(*args):
 521         """
 522         Extracts visitorData from an API response or ytcfg
 523         Appears to be used to track session state
 524         """
 525         return get_first(
 526             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 527             expected_type=str)
 528
 529     @functools.cached_property
 530     def is_authenticated(self):
 531         return bool(self._generate_sapisidhash_header())
 532
 533     def extract_ytcfg(self, video_id, webpage):
 534         if not webpage:
 535             return {}
 536         return self._parse_json(
 537             self._search_regex(
 538                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 539                 default='{}'), video_id, fatal=False) or {}
 540
 541     def generate_api_headers(
 542             self, *, ytcfg=None, account_syncid=None, session_index=None,
 543             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 544
 545         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 546         headers = {
 547             'X-YouTube-Client-Name': str(
 548                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 549             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 550             'Origin': origin,
 551             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 552             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 553             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 554         }
 555         if session_index is None:
 556             session_index = self._extract_session_index(ytcfg)
 557         if account_syncid or session_index is not None:
 558             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 559
 560         auth = self._generate_sapisidhash_header(origin)
 561         if auth is not None:
 562             headers['Authorization'] = auth
 563             headers['X-Origin'] = origin
 564         return {h: v for h, v in headers.items() if v is not None}
 565
 566     def _download_ytcfg(self, client, video_id):
 567         url = {
 568             'web': 'https://www.youtube.com',
 569             'web_music': 'https://music.youtube.com',
 570             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 571         }.get(client)
 572         if not url:
 573             return {}
 574         webpage = self._download_webpage(
 575             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 576         return self.extract_ytcfg(video_id, webpage) or {}
 577
 578     @staticmethod
 579     def _build_api_continuation_query(continuation, ctp=None):
 580         query = {
 581             'continuation': continuation
 582         }
 583         # TODO: Inconsistency with clickTrackingParams.
 584         # Currently we have a fixed ctp contained within context (from ytcfg)
 585         # and a ctp in root query for continuation.
 586         if ctp:
 587             query['clickTracking'] = {'clickTrackingParams': ctp}
 588         return query
 589
 590     @classmethod
 591     def _extract_next_continuation_data(cls, renderer):
 592         next_continuation = try_get(
 593             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 594                        lambda x: x['continuation']['reloadContinuationData']), dict)
 595         if not next_continuation:
 596             return
 597         continuation = next_continuation.get('continuation')
 598         if not continuation:
 599             return
 600         ctp = next_continuation.get('clickTrackingParams')
 601         return cls._build_api_continuation_query(continuation, ctp)
 602
 603     @classmethod
 604     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 605         if isinstance(continuation_ep, dict):
 606             continuation = try_get(
 607                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 608             if not continuation:
 609                 return
 610             ctp = continuation_ep.get('clickTrackingParams')
 611             return cls._build_api_continuation_query(continuation, ctp)
 612
 613     @classmethod
 614     def _extract_continuation(cls, renderer):
 615         next_continuation = cls._extract_next_continuation_data(renderer)
 616         if next_continuation:
 617             return next_continuation
 618
 619         contents = []
 620         for key in ('contents', 'items'):
 621             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 622
 623         for content in contents:
 624             if not isinstance(content, dict):
 625                 continue
 626             continuation_ep = try_get(
 627                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 628                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 629                 dict)
 630             continuation = cls._extract_continuation_ep_data(continuation_ep)
 631             if continuation:
 632                 return continuation
 633
 634     @classmethod
 635     def _extract_alerts(cls, data):
 636         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 637             if not isinstance(alert_dict, dict):
 638                 continue
 639             for alert in alert_dict.values():
 640                 alert_type = alert.get('type')
 641                 if not alert_type:
 642                     continue
 643                 message = cls._get_text(alert, 'text')
 644                 if message:
 645                     yield alert_type, message
 646
 647     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 648         errors = []
 649         warnings = []
 650         for alert_type, alert_message in alerts:
 651             if alert_type.lower() == 'error' and fatal:
 652                 errors.append([alert_type, alert_message])
 653             else:
 654                 warnings.append([alert_type, alert_message])
 655
 656         for alert_type, alert_message in (warnings + errors[:-1]):
 657             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 658         if errors:
 659             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 660
 661     def _extract_and_report_alerts(self, data, *args, **kwargs):
 662         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 663
 664     def _extract_badges(self, renderer: dict):
 665         badges = set()
 666         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 667             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], str)
 668             if label:
 669                 badges.add(label.lower())
 670         return badges
 671
 672     @staticmethod
 673     def _get_text(data, *path_list, max_runs=None):
 674         for path in path_list or [None]:
 675             if path is None:
 676                 obj = [data]
 677             else:
 678                 obj = traverse_obj(data, path, default=[])
 679                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 680                     obj = [obj]
 681             for item in obj:
 682                 text = try_get(item, lambda x: x['simpleText'], str)
 683                 if text:
 684                     return text
 685                 runs = try_get(item, lambda x: x['runs'], list) or []
 686                 if not runs and isinstance(item, list):
 687                     runs = item
 688
 689                 runs = runs[:min(len(runs), max_runs or len(runs))]
 690                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 691                 if text:
 692                     return text
 693
 694     def _get_count(self, data, *path_list):
 695         count_text = self._get_text(data, *path_list) or ''
 696         count = parse_count(count_text)
 697         if count is None:
 698             count = str_to_int(
 699                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 700         return count
 701
 702     @staticmethod
 703     def _extract_thumbnails(data, *path_list):
 704         """
 705         Extract thumbnails from thumbnails dict
 706         @param path_list: path list to level that contains 'thumbnails' key
 707         """
 708         thumbnails = []
 709         for path in path_list or [()]:
 710             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 711                 thumbnail_url = url_or_none(thumbnail.get('url'))
 712                 if not thumbnail_url:
 713                     continue
 714                 # Sometimes youtube gives a wrong thumbnail URL. See:
 715                 # https://github.com/yt-dlp/yt-dlp/issues/233
 716                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 717                 if 'maxresdefault' in thumbnail_url:
 718                     thumbnail_url = thumbnail_url.split('?')[0]
 719                 thumbnails.append({
 720                     'url': thumbnail_url,
 721                     'height': int_or_none(thumbnail.get('height')),
 722                     'width': int_or_none(thumbnail.get('width')),
 723                 })
 724         return thumbnails
 725
 726     @staticmethod
 727     def extract_relative_time(relative_time_text):
 728         """
 729         Extracts a relative time from string and converts to dt object
 730         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 731         """
 732         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 733         if mobj:
 734             start = mobj.group('start')
 735             if start:
 736                 return datetime_from_str(start)
 737             try:
 738                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 739             except ValueError:
 740                 return None
 741
 742     def _extract_time_text(self, renderer, *path_list):
 743         """@returns (timestamp, time_text)"""
 744         text = self._get_text(renderer, *path_list) or ''
 745         dt = self.extract_relative_time(text)
 746         timestamp = None
 747         if isinstance(dt, datetime.datetime):
 748             timestamp = calendar.timegm(dt.timetuple())
 749
 750         if timestamp is None:
 751             timestamp = (
 752                 unified_timestamp(text) or unified_timestamp(
 753                     self._search_regex(
 754                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 755                         text.lower(), 'time text', default=None)))
 756
 757         if text and timestamp is None:
 758             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 759         return timestamp, text
 760
 761     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 762                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 763                           default_client='web'):
 764         for retry in self.RetryManager():
 765             try:
 766                 response = self._call_api(
 767                     ep=ep, fatal=True, headers=headers,
 768                     video_id=item_id, query=query, note=note,
 769                     context=self._extract_context(ytcfg, default_client),
 770                     api_key=self._extract_api_key(ytcfg, default_client),
 771                     api_hostname=api_hostname, default_client=default_client)
 772             except ExtractorError as e:
 773                 if not isinstance(e.cause, network_exceptions):
 774                     return self._error_or_warning(e, fatal=fatal)
 775                 elif not isinstance(e.cause, urllib.error.HTTPError):
 776                     retry.error = e
 777                     continue
 778
 779                 first_bytes = e.cause.read(512)
 780                 if not is_html(first_bytes):
 781                     yt_error = try_get(
 782                         self._parse_json(
 783                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 784                         lambda x: x['error']['message'], str)
 785                     if yt_error:
 786                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 787                 # Downloading page may result in intermittent 5xx HTTP error
 788                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 789                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 790                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 791                 if e.cause.code not in (403, 429):
 792                     retry.error = e
 793                     continue
 794                 return self._error_or_warning(e, fatal=fatal)
 795
 796             try:
 797                 self._extract_and_report_alerts(response, only_once=True)
 798             except ExtractorError as e:
 799                 # YouTube servers may return errors we want to retry on in a 200 OK response
 800                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 801                 if 'unknown error' in e.msg.lower():
 802                     retry.error = e
 803                     continue
 804                 return self._error_or_warning(e, fatal=fatal)
 805             # Youtube sometimes sends incomplete data
 806             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 807             if not traverse_obj(response, *variadic(check_get_keys)):
 808                 retry.error = ExtractorError('Incomplete data received')
 809                 continue
 810
 811             return response
 812
 813     @staticmethod
 814     def is_music_url(url):
 815         return re.match(r'https?://music\.youtube\.com/', url) is not None
 816
 817     def _extract_video(self, renderer):
 818         video_id = renderer.get('videoId')
 819         title = self._get_text(renderer, 'title')
 820         description = self._get_text(renderer, 'descriptionSnippet')
 821         duration = parse_duration(self._get_text(
 822             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 823         if duration is None:
 824             duration = parse_duration(self._search_regex(
 825                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 826                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 827                 video_id, default=None, group='duration'))
 828
 829         view_count = self._get_count(renderer, 'viewCountText')
 830
 831         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 832         channel_id = traverse_obj(
 833             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 834             expected_type=str, get_all=False)
 835         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 836         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 837         overlay_style = traverse_obj(
 838             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 839             get_all=False, expected_type=str)
 840         badges = self._extract_badges(renderer)
 841         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 842         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 843             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 844             expected_type=str)) or ''
 845         url = f'https://www.youtube.com/watch?v={video_id}'
 846         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 847             url = f'https://www.youtube.com/shorts/{video_id}'
 848
 849         return {
 850             '_type': 'url',
 851             'ie_key': YoutubeIE.ie_key(),
 852             'id': video_id,
 853             'url': url,
 854             'title': title,
 855             'description': description,
 856             'duration': duration,
 857             'view_count': view_count,
 858             'uploader': uploader,
 859             'channel_id': channel_id,
 860             'thumbnails': thumbnails,
 861             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 862                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 863                             else None),
 864             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 865                             else 'was_live' if 'streamed' in time_text.lower()
 866                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 867                             else None),
 868             'release_timestamp': scheduled_timestamp,
 869             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 870         }
 871
 872
 873 class YoutubeIE(YoutubeBaseInfoExtractor):
 874     IE_DESC = 'YouTube'
 875     _VALID_URL = r"""(?x)^
 876                      (
 877                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 878                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 879                             (?:www\.)?deturl\.com/www\.youtube\.com|
 880                             (?:www\.)?pwnyoutube\.com|
 881                             (?:www\.)?hooktube\.com|
 882                             (?:www\.)?yourepeat\.com|
 883                             tube\.majestyc\.net|
 884                             %(invidious)s|
 885                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 886                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 887                          (?:                                                  # the various things that can precede the ID:
 888                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 889                              |(?:                                             # or the v= param in all its forms
 890                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 891                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 892                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 893                                  v=
 894                              )
 895                          ))
 896                          |(?:
 897                             youtu\.be|                                        # just youtu.be/xxxx
 898                             vid\.plus|                                        # or vid.plus/xxxx
 899                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 900                             %(invidious)s
 901                          )/
 902                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 903                          )
 904                      )?                                                       # all until now is optional -> you can pass the naked ID
 905                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 906                      (?(1).+)?                                                # if we found the ID, everything can follow
 907                      (?:\#|$)""" % {
 908         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 909     }
 910     _EMBED_REGEX = [r'''(?x)
 911         (?:
 912             <iframe[^>]+?src=|
 913             data-video-url=|
 914             <embed[^>]+?src=|
 915             embedSWF\(?:\s*|
 916             <object[^>]+data=|
 917             new\s+SWFObject\(
 918         )
 919         (["\'])
 920             (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
 921             (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
 922         \1''']
 923     _PLAYER_INFO_RE = (
 924         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 925         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 926         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 927     )
 928     _formats = {
 929         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 930         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 931         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 932         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 933         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 934         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 935         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 936         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 937         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 938         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 939         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 940         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 941         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 942         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 943         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 944         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 945         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 946         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 947
 948
 949         # 3D videos
 950         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 951         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 952         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 953         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 954         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 955         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 956         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 957
 958         # Apple HTTP Live Streaming
 959         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 960         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 961         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 962         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 963         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 964         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 965         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 966         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 967
 968         # DASH mp4 video
 969         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 970         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 971         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 972         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 973         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 974         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 975         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 976         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 977         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 978         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 979         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 980         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 981
 982         # Dash mp4 audio
 983         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 984         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 985         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 986         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 987         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 988         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 989         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 990
 991         # Dash webm
 992         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 993         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 994         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 995         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 996         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 997         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 998         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
 999         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1000         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1001         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1002         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1003         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1004         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1005         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1006         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1007         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1008         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1009         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1010         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1011         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1012         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1013         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1014
1015         # Dash webm audio
1016         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1017         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1018
1019         # Dash webm audio with opus inside
1020         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1021         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1022         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1023
1024         # RTMP (unnamed)
1025         '_rtmp': {'protocol': 'rtmp'},
1026
1027         # av01 video only formats sometimes served with "unknown" codecs
1028         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1029         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1030         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1031         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1032         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1033         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1034         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1035         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1036     }
1037     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1038
1039     _GEO_BYPASS = False
1040
1041     IE_NAME = 'youtube'
1042     _TESTS = [
1043         {
1044             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1045             'info_dict': {
1046                 'id': 'BaW_jenozKc',
1047                 'ext': 'mp4',
1048                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1049                 'uploader': 'Philipp Hagemeister',
1050                 'uploader_id': 'phihag',
1051                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1052                 'channel': 'Philipp Hagemeister',
1053                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1054                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1055                 'upload_date': '20121002',
1056                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1057                 'categories': ['Science & Technology'],
1058                 'tags': ['youtube-dl'],
1059                 'duration': 10,
1060                 'view_count': int,
1061                 'like_count': int,
1062                 'availability': 'public',
1063                 'playable_in_embed': True,
1064                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1065                 'live_status': 'not_live',
1066                 'age_limit': 0,
1067                 'start_time': 1,
1068                 'end_time': 9,
1069                 'comment_count': int,
1070                 'channel_follower_count': int
1071             }
1072         },
1073         {
1074             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1075             'note': 'Embed-only video (#1746)',
1076             'info_dict': {
1077                 'id': 'yZIXLfi8CZQ',
1078                 'ext': 'mp4',
1079                 'upload_date': '20120608',
1080                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1081                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1082                 'uploader': 'SET India',
1083                 'uploader_id': 'setindia',
1084                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1085                 'age_limit': 18,
1086             },
1087             'skip': 'Private video',
1088         },
1089         {
1090             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1091             'note': 'Use the first video ID in the URL',
1092             'info_dict': {
1093                 'id': 'BaW_jenozKc',
1094                 'ext': 'mp4',
1095                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1096                 'uploader': 'Philipp Hagemeister',
1097                 'uploader_id': 'phihag',
1098                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1099                 'channel': 'Philipp Hagemeister',
1100                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1101                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1102                 'upload_date': '20121002',
1103                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1104                 'categories': ['Science & Technology'],
1105                 'tags': ['youtube-dl'],
1106                 'duration': 10,
1107                 'view_count': int,
1108                 'like_count': int,
1109                 'availability': 'public',
1110                 'playable_in_embed': True,
1111                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1112                 'live_status': 'not_live',
1113                 'age_limit': 0,
1114                 'comment_count': int,
1115                 'channel_follower_count': int
1116             },
1117             'params': {
1118                 'skip_download': True,
1119             },
1120         },
1121         {
1122             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1123             'note': '256k DASH audio (format 141) via DASH manifest',
1124             'info_dict': {
1125                 'id': 'a9LDPn-MO4I',
1126                 'ext': 'm4a',
1127                 'upload_date': '20121002',
1128                 'uploader_id': '8KVIDEO',
1129                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1130                 'description': '',
1131                 'uploader': '8KVIDEO',
1132                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1133             },
1134             'params': {
1135                 'youtube_include_dash_manifest': True,
1136                 'format': '141',
1137             },
1138             'skip': 'format 141 not served anymore',
1139         },
1140         # DASH manifest with encrypted signature
1141         {
1142             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1143             'info_dict': {
1144                 'id': 'IB3lcPjvWLA',
1145                 'ext': 'm4a',
1146                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1147                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1148                 'duration': 244,
1149                 'uploader': 'AfrojackVEVO',
1150                 'uploader_id': 'AfrojackVEVO',
1151                 'upload_date': '20131011',
1152                 'abr': 129.495,
1153                 'like_count': int,
1154                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1155                 'playable_in_embed': True,
1156                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1157                 'view_count': int,
1158                 'track': 'The Spark',
1159                 'live_status': 'not_live',
1160                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1161                 'channel': 'Afrojack',
1162                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1163                 'tags': 'count:19',
1164                 'availability': 'public',
1165                 'categories': ['Music'],
1166                 'age_limit': 0,
1167                 'alt_title': 'The Spark',
1168                 'channel_follower_count': int
1169             },
1170             'params': {
1171                 'youtube_include_dash_manifest': True,
1172                 'format': '141/bestaudio[ext=m4a]',
1173             },
1174         },
1175         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1176         {
1177             'note': 'Embed allowed age-gate video',
1178             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1179             'info_dict': {
1180                 'id': 'HtVdAasjOgU',
1181                 'ext': 'mp4',
1182                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1183                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1184                 'duration': 142,
1185                 'uploader': 'The Witcher',
1186                 'uploader_id': 'WitcherGame',
1187                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1188                 'upload_date': '20140605',
1189                 'age_limit': 18,
1190                 'categories': ['Gaming'],
1191                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1192                 'availability': 'needs_auth',
1193                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1194                 'like_count': int,
1195                 'channel': 'The Witcher',
1196                 'live_status': 'not_live',
1197                 'tags': 'count:17',
1198                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1199                 'playable_in_embed': True,
1200                 'view_count': int,
1201                 'channel_follower_count': int
1202             },
1203         },
1204         {
1205             'note': 'Age-gate video with embed allowed in public site',
1206             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1207             'info_dict': {
1208                 'id': 'HsUATh_Nc2U',
1209                 'ext': 'mp4',
1210                 'title': 'Godzilla 2 (Official Video)',
1211                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1212                 'upload_date': '20200408',
1213                 'uploader_id': 'FlyingKitty900',
1214                 'uploader': 'FlyingKitty',
1215                 'age_limit': 18,
1216                 'availability': 'needs_auth',
1217                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1218                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1219                 'channel': 'FlyingKitty',
1220                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1221                 'view_count': int,
1222                 'categories': ['Entertainment'],
1223                 'live_status': 'not_live',
1224                 'tags': ['Flyingkitty', 'godzilla 2'],
1225                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1226                 'like_count': int,
1227                 'duration': 177,
1228                 'playable_in_embed': True,
1229                 'channel_follower_count': int
1230             },
1231         },
1232         {
1233             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1234             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1235             'info_dict': {
1236                 'id': 'Tq92D6wQ1mg',
1237                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1238                 'ext': 'mp4',
1239                 'upload_date': '20191228',
1240                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1241                 'uploader': 'Projekt Melody',
1242                 'description': 'md5:17eccca93a786d51bc67646756894066',
1243                 'age_limit': 18,
1244                 'like_count': int,
1245                 'availability': 'needs_auth',
1246                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1247                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1248                 'view_count': int,
1249                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1250                 'channel': 'Projekt Melody',
1251                 'live_status': 'not_live',
1252                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1253                 'playable_in_embed': True,
1254                 'categories': ['Entertainment'],
1255                 'duration': 106,
1256                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1257                 'comment_count': int,
1258                 'channel_follower_count': int
1259             },
1260         },
1261         {
1262             'note': 'Non-Agegated non-embeddable video',
1263             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1264             'info_dict': {
1265                 'id': 'MeJVWBSsPAY',
1266                 'ext': 'mp4',
1267                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1268                 'uploader': 'Herr Lurik',
1269                 'uploader_id': 'st3in234',
1270                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1271                 'upload_date': '20130730',
1272                 'track': 'Such mich find mich',
1273                 'age_limit': 0,
1274                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1275                 'like_count': int,
1276                 'playable_in_embed': False,
1277                 'creator': 'OOMPH!',
1278                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1279                 'view_count': int,
1280                 'alt_title': 'Such mich find mich',
1281                 'duration': 210,
1282                 'channel': 'Herr Lurik',
1283                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1284                 'categories': ['Music'],
1285                 'availability': 'public',
1286                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1287                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1288                 'live_status': 'not_live',
1289                 'artist': 'OOMPH!',
1290                 'channel_follower_count': int
1291             },
1292         },
1293         {
1294             'note': 'Non-bypassable age-gated video',
1295             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1296             'only_matching': True,
1297         },
1298         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1299         # YouTube Red ad is not captured for creator
1300         {
1301             'url': '__2ABJjxzNo',
1302             'info_dict': {
1303                 'id': '__2ABJjxzNo',
1304                 'ext': 'mp4',
1305                 'duration': 266,
1306                 'upload_date': '20100430',
1307                 'uploader_id': 'deadmau5',
1308                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1309                 'creator': 'deadmau5',
1310                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1311                 'uploader': 'deadmau5',
1312                 'title': 'Deadmau5 - Some Chords (HD)',
1313                 'alt_title': 'Some Chords',
1314                 'availability': 'public',
1315                 'tags': 'count:14',
1316                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1317                 'view_count': int,
1318                 'live_status': 'not_live',
1319                 'channel': 'deadmau5',
1320                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1321                 'like_count': int,
1322                 'track': 'Some Chords',
1323                 'artist': 'deadmau5',
1324                 'playable_in_embed': True,
1325                 'age_limit': 0,
1326                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1327                 'categories': ['Music'],
1328                 'album': 'Some Chords',
1329                 'channel_follower_count': int
1330             },
1331             'expected_warnings': [
1332                 'DASH manifest missing',
1333             ]
1334         },
1335         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1336         {
1337             'url': 'lqQg6PlCWgI',
1338             'info_dict': {
1339                 'id': 'lqQg6PlCWgI',
1340                 'ext': 'mp4',
1341                 'duration': 6085,
1342                 'upload_date': '20150827',
1343                 'uploader_id': 'olympic',
1344                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1345                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1346                 'uploader': 'Olympics',
1347                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1348                 'like_count': int,
1349                 'release_timestamp': 1343767800,
1350                 'playable_in_embed': True,
1351                 'categories': ['Sports'],
1352                 'release_date': '20120731',
1353                 'channel': 'Olympics',
1354                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1355                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1356                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1357                 'age_limit': 0,
1358                 'availability': 'public',
1359                 'live_status': 'was_live',
1360                 'view_count': int,
1361                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1362                 'channel_follower_count': int
1363             },
1364             'params': {
1365                 'skip_download': 'requires avconv',
1366             }
1367         },
1368         # Non-square pixels
1369         {
1370             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1371             'info_dict': {
1372                 'id': '_b-2C3KPAM0',
1373                 'ext': 'mp4',
1374                 'stretched_ratio': 16 / 9.,
1375                 'duration': 85,
1376                 'upload_date': '20110310',
1377                 'uploader_id': 'AllenMeow',
1378                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1379                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1380                 'uploader': '孫ᄋᄅ',
1381                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1382                 'playable_in_embed': True,
1383                 'channel': '孫ᄋᄅ',
1384                 'age_limit': 0,
1385                 'tags': 'count:11',
1386                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1387                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1388                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1389                 'view_count': int,
1390                 'categories': ['People & Blogs'],
1391                 'like_count': int,
1392                 'live_status': 'not_live',
1393                 'availability': 'unlisted',
1394                 'comment_count': int,
1395                 'channel_follower_count': int
1396             },
1397         },
1398         # url_encoded_fmt_stream_map is empty string
1399         {
1400             'url': 'qEJwOuvDf7I',
1401             'info_dict': {
1402                 'id': 'qEJwOuvDf7I',
1403                 'ext': 'webm',
1404                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1405                 'description': '',
1406                 'upload_date': '20150404',
1407                 'uploader_id': 'spbelect',
1408                 'uploader': 'Наблюдатели Петербурга',
1409             },
1410             'params': {
1411                 'skip_download': 'requires avconv',
1412             },
1413             'skip': 'This live event has ended.',
1414         },
1415         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1416         {
1417             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1418             'info_dict': {
1419                 'id': 'FIl7x6_3R5Y',
1420                 'ext': 'webm',
1421                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1422                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1423                 'duration': 220,
1424                 'upload_date': '20150625',
1425                 'uploader_id': 'dorappi2000',
1426                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1427                 'uploader': 'dorappi2000',
1428                 'formats': 'mincount:31',
1429             },
1430             'skip': 'not actual anymore',
1431         },
1432         # DASH manifest with segment_list
1433         {
1434             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1435             'md5': '8ce563a1d667b599d21064e982ab9e31',
1436             'info_dict': {
1437                 'id': 'CsmdDsKjzN8',
1438                 'ext': 'mp4',
1439                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1440                 'uploader': 'Airtek',
1441                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1442                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1443                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1444             },
1445             'params': {
1446                 'youtube_include_dash_manifest': True,
1447                 'format': '135',  # bestvideo
1448             },
1449             'skip': 'This live event has ended.',
1450         },
1451         {
1452             # Multifeed videos (multiple cameras), URL is for Main Camera
1453             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1454             'info_dict': {
1455                 'id': 'jvGDaLqkpTg',
1456                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1457                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1458             },
1459             'playlist': [{
1460                 'info_dict': {
1461                     'id': 'jvGDaLqkpTg',
1462                     'ext': 'mp4',
1463                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1464                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1465                     'duration': 10643,
1466                     'upload_date': '20161111',
1467                     'uploader': 'Team PGP',
1468                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1469                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1470                 },
1471             }, {
1472                 'info_dict': {
1473                     'id': '3AKt1R1aDnw',
1474                     'ext': 'mp4',
1475                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1476                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1477                     'duration': 10991,
1478                     'upload_date': '20161111',
1479                     'uploader': 'Team PGP',
1480                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1481                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1482                 },
1483             }, {
1484                 'info_dict': {
1485                     'id': 'RtAMM00gpVc',
1486                     'ext': 'mp4',
1487                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1488                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1489                     'duration': 10995,
1490                     'upload_date': '20161111',
1491                     'uploader': 'Team PGP',
1492                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1493                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1494                 },
1495             }, {
1496                 'info_dict': {
1497                     'id': '6N2fdlP3C5U',
1498                     'ext': 'mp4',
1499                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1500                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1501                     'duration': 10990,
1502                     'upload_date': '20161111',
1503                     'uploader': 'Team PGP',
1504                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1505                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1506                 },
1507             }],
1508             'params': {
1509                 'skip_download': True,
1510             },
1511             'skip': 'Not multifeed anymore',
1512         },
1513         {
1514             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1515             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1516             'info_dict': {
1517                 'id': 'gVfLd0zydlo',
1518                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1519             },
1520             'playlist_count': 2,
1521             'skip': 'Not multifeed anymore',
1522         },
1523         {
1524             'url': 'https://vid.plus/FlRa-iH7PGw',
1525             'only_matching': True,
1526         },
1527         {
1528             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1529             'only_matching': True,
1530         },
1531         {
1532             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1533             # Also tests cut-off URL expansion in video description (see
1534             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1535             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1536             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1537             'info_dict': {
1538                 'id': 'lsguqyKfVQg',
1539                 'ext': 'mp4',
1540                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1541                 'alt_title': 'Dark Walk',
1542                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1543                 'duration': 133,
1544                 'upload_date': '20151119',
1545                 'uploader_id': 'IronSoulElf',
1546                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1547                 'uploader': 'IronSoulElf',
1548                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1549                 'track': 'Dark Walk',
1550                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1551                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1552                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1553                 'categories': ['Film & Animation'],
1554                 'view_count': int,
1555                 'live_status': 'not_live',
1556                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1557                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1558                 'tags': 'count:13',
1559                 'availability': 'public',
1560                 'channel': 'IronSoulElf',
1561                 'playable_in_embed': True,
1562                 'like_count': int,
1563                 'age_limit': 0,
1564                 'channel_follower_count': int
1565             },
1566             'params': {
1567                 'skip_download': True,
1568             },
1569         },
1570         {
1571             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1572             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1573             'only_matching': True,
1574         },
1575         {
1576             # Video with yt:stretch=17:0
1577             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1578             'info_dict': {
1579                 'id': 'Q39EVAstoRM',
1580                 'ext': 'mp4',
1581                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1582                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1583                 'upload_date': '20151107',
1584                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1585                 'uploader': 'CH GAMER DROID',
1586             },
1587             'params': {
1588                 'skip_download': True,
1589             },
1590             'skip': 'This video does not exist.',
1591         },
1592         {
1593             # Video with incomplete 'yt:stretch=16:'
1594             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1595             'only_matching': True,
1596         },
1597         {
1598             # Video licensed under Creative Commons
1599             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1600             'info_dict': {
1601                 'id': 'M4gD1WSo5mA',
1602                 'ext': 'mp4',
1603                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1604                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1605                 'duration': 721,
1606                 'upload_date': '20150128',
1607                 'uploader_id': 'BerkmanCenter',
1608                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1609                 'uploader': 'The Berkman Klein Center for Internet & Society',
1610                 'license': 'Creative Commons Attribution license (reuse allowed)',
1611                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1612                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1613                 'like_count': int,
1614                 'age_limit': 0,
1615                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1616                 'channel': 'The Berkman Klein Center for Internet & Society',
1617                 'availability': 'public',
1618                 'view_count': int,
1619                 'categories': ['Education'],
1620                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1621                 'live_status': 'not_live',
1622                 'playable_in_embed': True,
1623                 'comment_count': int,
1624                 'channel_follower_count': int
1625             },
1626             'params': {
1627                 'skip_download': True,
1628             },
1629         },
1630         {
1631             # Channel-like uploader_url
1632             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1633             'info_dict': {
1634                 'id': 'eQcmzGIKrzg',
1635                 'ext': 'mp4',
1636                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1637                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1638                 'duration': 4060,
1639                 'upload_date': '20151120',
1640                 'uploader': 'Bernie Sanders',
1641                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1642                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1643                 'license': 'Creative Commons Attribution license (reuse allowed)',
1644                 'playable_in_embed': True,
1645                 'tags': 'count:12',
1646                 'like_count': int,
1647                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1648                 'age_limit': 0,
1649                 'availability': 'public',
1650                 'categories': ['News & Politics'],
1651                 'channel': 'Bernie Sanders',
1652                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1653                 'view_count': int,
1654                 'live_status': 'not_live',
1655                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1656                 'comment_count': int,
1657                 'channel_follower_count': int
1658             },
1659             'params': {
1660                 'skip_download': True,
1661             },
1662         },
1663         {
1664             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1665             'only_matching': True,
1666         },
1667         {
1668             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1669             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1670             'only_matching': True,
1671         },
1672         {
1673             # Rental video preview
1674             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1675             'info_dict': {
1676                 'id': 'uGpuVWrhIzE',
1677                 'ext': 'mp4',
1678                 'title': 'Piku - Trailer',
1679                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1680                 'upload_date': '20150811',
1681                 'uploader': 'FlixMatrix',
1682                 'uploader_id': 'FlixMatrixKaravan',
1683                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1684                 'license': 'Standard YouTube License',
1685             },
1686             'params': {
1687                 'skip_download': True,
1688             },
1689             'skip': 'This video is not available.',
1690         },
1691         {
1692             # YouTube Red video with episode data
1693             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1694             'info_dict': {
1695                 'id': 'iqKdEhx-dD4',
1696                 'ext': 'mp4',
1697                 'title': 'Isolation - Mind Field (Ep 1)',
1698                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1699                 'duration': 2085,
1700                 'upload_date': '20170118',
1701                 'uploader': 'Vsauce',
1702                 'uploader_id': 'Vsauce',
1703                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1704                 'series': 'Mind Field',
1705                 'season_number': 1,
1706                 'episode_number': 1,
1707                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1708                 'tags': 'count:12',
1709                 'view_count': int,
1710                 'availability': 'public',
1711                 'age_limit': 0,
1712                 'channel': 'Vsauce',
1713                 'episode': 'Episode 1',
1714                 'categories': ['Entertainment'],
1715                 'season': 'Season 1',
1716                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1717                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1718                 'like_count': int,
1719                 'playable_in_embed': True,
1720                 'live_status': 'not_live',
1721                 'channel_follower_count': int
1722             },
1723             'params': {
1724                 'skip_download': True,
1725             },
1726             'expected_warnings': [
1727                 'Skipping DASH manifest',
1728             ],
1729         },
1730         {
1731             # The following content has been identified by the YouTube community
1732             # as inappropriate or offensive to some audiences.
1733             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1734             'info_dict': {
1735                 'id': '6SJNVb0GnPI',
1736                 'ext': 'mp4',
1737                 'title': 'Race Differences in Intelligence',
1738                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1739                 'duration': 965,
1740                 'upload_date': '20140124',
1741                 'uploader': 'New Century Foundation',
1742                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1743                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1744             },
1745             'params': {
1746                 'skip_download': True,
1747             },
1748             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1749         },
1750         {
1751             # itag 212
1752             'url': '1t24XAntNCY',
1753             'only_matching': True,
1754         },
1755         {
1756             # geo restricted to JP
1757             'url': 'sJL6WA-aGkQ',
1758             'only_matching': True,
1759         },
1760         {
1761             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1762             'only_matching': True,
1763         },
1764         {
1765             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1766             'only_matching': True,
1767         },
1768         {
1769             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1770             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1771             'only_matching': True,
1772         },
1773         {
1774             # DRM protected
1775             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1776             'only_matching': True,
1777         },
1778         {
1779             # Video with unsupported adaptive stream type formats
1780             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1781             'info_dict': {
1782                 'id': 'Z4Vy8R84T1U',
1783                 'ext': 'mp4',
1784                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1785                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1786                 'duration': 433,
1787                 'upload_date': '20130923',
1788                 'uploader': 'Amelia Putri Harwita',
1789                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1790                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1791                 'formats': 'maxcount:10',
1792             },
1793             'params': {
1794                 'skip_download': True,
1795                 'youtube_include_dash_manifest': False,
1796             },
1797             'skip': 'not actual anymore',
1798         },
1799         {
1800             # Youtube Music Auto-generated description
1801             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1802             'info_dict': {
1803                 'id': 'MgNrAu2pzNs',
1804                 'ext': 'mp4',
1805                 'title': 'Voyeur Girl',
1806                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1807                 'upload_date': '20190312',
1808                 'uploader': 'Stephen - Topic',
1809                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1810                 'artist': 'Stephen',
1811                 'track': 'Voyeur Girl',
1812                 'album': 'it\'s too much love to know my dear',
1813                 'release_date': '20190313',
1814                 'release_year': 2019,
1815                 'alt_title': 'Voyeur Girl',
1816                 'view_count': int,
1817                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1818                 'playable_in_embed': True,
1819                 'like_count': int,
1820                 'categories': ['Music'],
1821                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1822                 'channel': 'Stephen',
1823                 'availability': 'public',
1824                 'creator': 'Stephen',
1825                 'duration': 169,
1826                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1827                 'age_limit': 0,
1828                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1829                 'tags': 'count:11',
1830                 'live_status': 'not_live',
1831                 'channel_follower_count': int
1832             },
1833             'params': {
1834                 'skip_download': True,
1835             },
1836         },
1837         {
1838             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1839             'only_matching': True,
1840         },
1841         {
1842             # invalid -> valid video id redirection
1843             'url': 'DJztXj2GPfl',
1844             'info_dict': {
1845                 'id': 'DJztXj2GPfk',
1846                 'ext': 'mp4',
1847                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1848                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1849                 'upload_date': '20090125',
1850                 'uploader': 'Prochorowka',
1851                 'uploader_id': 'Prochorowka',
1852                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1853                 'artist': 'Panjabi MC',
1854                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1855                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1856             },
1857             'params': {
1858                 'skip_download': True,
1859             },
1860             'skip': 'Video unavailable',
1861         },
1862         {
1863             # empty description results in an empty string
1864             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1865             'info_dict': {
1866                 'id': 'x41yOUIvK2k',
1867                 'ext': 'mp4',
1868                 'title': 'IMG 3456',
1869                 'description': '',
1870                 'upload_date': '20170613',
1871                 'uploader_id': 'ElevageOrVert',
1872                 'uploader': 'ElevageOrVert',
1873                 'view_count': int,
1874                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1875                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1876                 'like_count': int,
1877                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1878                 'tags': [],
1879                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1880                 'availability': 'public',
1881                 'age_limit': 0,
1882                 'categories': ['Pets & Animals'],
1883                 'duration': 7,
1884                 'playable_in_embed': True,
1885                 'live_status': 'not_live',
1886                 'channel': 'ElevageOrVert',
1887                 'channel_follower_count': int
1888             },
1889             'params': {
1890                 'skip_download': True,
1891             },
1892         },
1893         {
1894             # with '};' inside yt initial data (see [1])
1895             # see [2] for an example with '};' inside ytInitialPlayerResponse
1896             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1897             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1898             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1899             'info_dict': {
1900                 'id': 'CHqg6qOn4no',
1901                 'ext': 'mp4',
1902                 'title': 'Part 77   Sort a list of simple types in c#',
1903                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1904                 'upload_date': '20130831',
1905                 'uploader_id': 'kudvenkat',
1906                 'uploader': 'kudvenkat',
1907                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1908                 'like_count': int,
1909                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1910                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1911                 'live_status': 'not_live',
1912                 'categories': ['Education'],
1913                 'availability': 'public',
1914                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1915                 'tags': 'count:12',
1916                 'playable_in_embed': True,
1917                 'age_limit': 0,
1918                 'view_count': int,
1919                 'duration': 522,
1920                 'channel': 'kudvenkat',
1921                 'comment_count': int,
1922                 'channel_follower_count': int
1923             },
1924             'params': {
1925                 'skip_download': True,
1926             },
1927         },
1928         {
1929             # another example of '};' in ytInitialData
1930             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1931             'only_matching': True,
1932         },
1933         {
1934             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1935             'only_matching': True,
1936         },
1937         {
1938             # https://github.com/ytdl-org/youtube-dl/pull/28094
1939             'url': 'OtqTfy26tG0',
1940             'info_dict': {
1941                 'id': 'OtqTfy26tG0',
1942                 'ext': 'mp4',
1943                 'title': 'Burn Out',
1944                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1945                 'upload_date': '20141120',
1946                 'uploader': 'The Cinematic Orchestra - Topic',
1947                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1948                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1949                 'artist': 'The Cinematic Orchestra',
1950                 'track': 'Burn Out',
1951                 'album': 'Every Day',
1952                 'like_count': int,
1953                 'live_status': 'not_live',
1954                 'alt_title': 'Burn Out',
1955                 'duration': 614,
1956                 'age_limit': 0,
1957                 'view_count': int,
1958                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1959                 'creator': 'The Cinematic Orchestra',
1960                 'channel': 'The Cinematic Orchestra',
1961                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1962                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1963                 'availability': 'public',
1964                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1965                 'categories': ['Music'],
1966                 'playable_in_embed': True,
1967                 'channel_follower_count': int
1968             },
1969             'params': {
1970                 'skip_download': True,
1971             },
1972         },
1973         {
1974             # controversial video, only works with bpctr when authenticated with cookies
1975             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1976             'only_matching': True,
1977         },
1978         {
1979             # controversial video, requires bpctr/contentCheckOk
1980             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1981             'info_dict': {
1982                 'id': 'SZJvDhaSDnc',
1983                 'ext': 'mp4',
1984                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1985                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1986                 'uploader': 'CBS Mornings',
1987                 'uploader_id': 'CBSThisMorning',
1988                 'upload_date': '20140716',
1989                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1990                 'duration': 170,
1991                 'categories': ['News & Politics'],
1992                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
1993                 'view_count': int,
1994                 'channel': 'CBS Mornings',
1995                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
1996                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
1997                 'age_limit': 18,
1998                 'availability': 'needs_auth',
1999                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2000                 'like_count': int,
2001                 'live_status': 'not_live',
2002                 'playable_in_embed': True,
2003                 'channel_follower_count': int
2004             }
2005         },
2006         {
2007             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2008             'url': 'cBvYw8_A0vQ',
2009             'info_dict': {
2010                 'id': 'cBvYw8_A0vQ',
2011                 'ext': 'mp4',
2012                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2013                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2014                 'upload_date': '20201120',
2015                 'uploader': 'Walk around Japan',
2016                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2017                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2018                 'duration': 1456,
2019                 'categories': ['Travel & Events'],
2020                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2021                 'view_count': int,
2022                 'channel': 'Walk around Japan',
2023                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2024                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2025                 'age_limit': 0,
2026                 'availability': 'public',
2027                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2028                 'live_status': 'not_live',
2029                 'playable_in_embed': True,
2030                 'channel_follower_count': int
2031             },
2032             'params': {
2033                 'skip_download': True,
2034             },
2035         }, {
2036             # Has multiple audio streams
2037             'url': 'WaOKSUlf4TM',
2038             'only_matching': True
2039         }, {
2040             # Requires Premium: has format 141 when requested using YTM url
2041             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2042             'only_matching': True
2043         }, {
2044             # multiple subtitles with same lang_code
2045             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2046             'only_matching': True,
2047         }, {
2048             # Force use android client fallback
2049             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2050             'info_dict': {
2051                 'id': 'YOelRv7fMxY',
2052                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2053                 'ext': '3gp',
2054                 'upload_date': '20210624',
2055                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2056                 'uploader': 'colinfurze',
2057                 'uploader_id': 'colinfurze',
2058                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2059                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2060                 'duration': 596,
2061                 'categories': ['Entertainment'],
2062                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2063                 'view_count': int,
2064                 'channel': 'colinfurze',
2065                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2066                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2067                 'age_limit': 0,
2068                 'availability': 'public',
2069                 'like_count': int,
2070                 'live_status': 'not_live',
2071                 'playable_in_embed': True,
2072                 'channel_follower_count': int
2073             },
2074             'params': {
2075                 'format': '17',  # 3gp format available on android
2076                 'extractor_args': {'youtube': {'player_client': ['android']}},
2077             },
2078         },
2079         {
2080             # Skip download of additional client configs (remix client config in this case)
2081             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2082             'only_matching': True,
2083             'params': {
2084                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2085             },
2086         }, {
2087             # shorts
2088             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2089             'only_matching': True,
2090         }, {
2091             'note': 'Storyboards',
2092             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2093             'info_dict': {
2094                 'id': '5KLPxDtMqe8',
2095                 'ext': 'mhtml',
2096                 'format_id': 'sb0',
2097                 'title': 'Your Brain is Plastic',
2098                 'uploader_id': 'scishow',
2099                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2100                 'upload_date': '20140324',
2101                 'uploader': 'SciShow',
2102                 'like_count': int,
2103                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2104                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2105                 'view_count': int,
2106                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2107                 'playable_in_embed': True,
2108                 'tags': 'count:12',
2109                 'uploader_url': 'http://www.youtube.com/user/scishow',
2110                 'availability': 'public',
2111                 'channel': 'SciShow',
2112                 'live_status': 'not_live',
2113                 'duration': 248,
2114                 'categories': ['Education'],
2115                 'age_limit': 0,
2116                 'channel_follower_count': int
2117             }, 'params': {'format': 'mhtml', 'skip_download': True}
2118         }, {
2119             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2120             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2121             'info_dict': {
2122                 'id': '2NUZ8W2llS4',
2123                 'ext': 'mp4',
2124                 'title': 'The NP that test your phone performance 🙂',
2125                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2126                 'uploader': 'Leon Nguyen',
2127                 'uploader_id': 'VNSXIII',
2128                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2129                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2130                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2131                 'duration': 21,
2132                 'view_count': int,
2133                 'age_limit': 0,
2134                 'categories': ['Gaming'],
2135                 'tags': 'count:23',
2136                 'playable_in_embed': True,
2137                 'live_status': 'not_live',
2138                 'upload_date': '20220103',
2139                 'like_count': int,
2140                 'availability': 'public',
2141                 'channel': 'Leon Nguyen',
2142                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2143                 'comment_count': int,
2144                 'channel_follower_count': int
2145             }
2146         }, {
2147             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2148             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2149             'info_dict': {
2150                 'id': 'mzZzzBU6lrM',
2151                 'ext': 'mp4',
2152                 'title': 'I Met GeorgeNotFound In Real Life...',
2153                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2154                 'uploader': 'Quackity',
2155                 'uploader_id': 'QuackityHQ',
2156                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2157                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2158                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2159                 'duration': 955,
2160                 'view_count': int,
2161                 'age_limit': 0,
2162                 'categories': ['Entertainment'],
2163                 'tags': 'count:26',
2164                 'playable_in_embed': True,
2165                 'live_status': 'not_live',
2166                 'release_timestamp': 1641172509,
2167                 'release_date': '20220103',
2168                 'upload_date': '20220103',
2169                 'like_count': int,
2170                 'availability': 'public',
2171                 'channel': 'Quackity',
2172                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2173                 'channel_follower_count': int
2174             }
2175         },
2176         {   # continuous livestream. Microformat upload date should be preferred.
2177             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2178             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2179             'info_dict': {
2180                 'id': 'kgx4WGK0oNU',
2181                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2182                 'ext': 'mp4',
2183                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2184                 'availability': 'public',
2185                 'age_limit': 0,
2186                 'release_timestamp': 1637975704,
2187                 'upload_date': '20210619',
2188                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2189                 'live_status': 'is_live',
2190                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2191                 'uploader': '阿鲍Abao',
2192                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2193                 'channel': 'Abao in Tokyo',
2194                 'channel_follower_count': int,
2195                 'release_date': '20211127',
2196                 'tags': 'count:39',
2197                 'categories': ['People & Blogs'],
2198                 'like_count': int,
2199                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2200                 'view_count': int,
2201                 'playable_in_embed': True,
2202                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2203             },
2204             'params': {'skip_download': True}
2205         }, {
2206             # Story. Requires specific player params to work.
2207             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2208             'info_dict': {
2209                 'id': 'vv8qTUWmulI',
2210                 'ext': 'mp4',
2211                 'availability': 'unlisted',
2212                 'view_count': int,
2213                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2214                 'upload_date': '20220526',
2215                 'categories': ['Education'],
2216                 'title': 'Story',
2217                 'channel': 'IT\'S HISTORY',
2218                 'description': '',
2219                 'uploader_id': 'BlastfromthePast',
2220                 'duration': 12,
2221                 'uploader': 'IT\'S HISTORY',
2222                 'playable_in_embed': True,
2223                 'age_limit': 0,
2224                 'live_status': 'not_live',
2225                 'tags': [],
2226                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2227                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2228                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2229             },
2230             'skip': 'stories get removed after some period of time',
2231         }, {
2232             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2233             'info_dict': {
2234                 'id': 'tjjjtzRLHvA',
2235                 'ext': 'mp4',
2236                 'title': 'ハッシュタグ無し };if window.ytcsi',
2237                 'upload_date': '20220323',
2238                 'like_count': int,
2239                 'availability': 'unlisted',
2240                 'channel': 'nao20010128nao',
2241                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2242                 'age_limit': 0,
2243                 'uploader': 'nao20010128nao',
2244                 'uploader_id': 'nao20010128nao',
2245                 'categories': ['Music'],
2246                 'view_count': int,
2247                 'description': '',
2248                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2249                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2250                 'live_status': 'not_live',
2251                 'playable_in_embed': True,
2252                 'channel_follower_count': int,
2253                 'duration': 6,
2254                 'tags': [],
2255                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2256             }
2257         }
2258     ]
2259
2260     _WEBPAGE_TESTS = [
2261         # YouTube <object> embed
2262         {
2263             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2264             'md5': '873c81d308b979f0e23ee7e620b312a3',
2265             'info_dict': {
2266                 'id': 'msN87y-iEx0',
2267                 'ext': 'mp4',
2268                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2269                 'upload_date': '20080526',
2270                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2271                 'uploader': 'Christopher Sykes',
2272                 'uploader_id': 'ChristopherJSykes',
2273                 'age_limit': 0,
2274                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2275                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2276                 'playable_in_embed': True,
2277                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2278                 'like_count': int,
2279                 'comment_count': int,
2280                 'channel': 'Christopher Sykes',
2281                 'live_status': 'not_live',
2282                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2283                 'availability': 'public',
2284                 'duration': 195,
2285                 'view_count': int,
2286                 'categories': ['Science & Technology'],
2287                 'channel_follower_count': int,
2288                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2289             },
2290             'params': {
2291                 'skip_download': True,
2292             }
2293         },
2294     ]
2295
2296     @classmethod
2297     def suitable(cls, url):
2298         from ..utils import parse_qs
2299
2300         qs = parse_qs(url)
2301         if qs.get('list', [None])[0]:
2302             return False
2303         return super().suitable(url)
2304
2305     def __init__(self, *args, **kwargs):
2306         super().__init__(*args, **kwargs)
2307         self._code_cache = {}
2308         self._player_cache = {}
2309
2310     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2311         lock = threading.Lock()
2312
2313         is_live = True
2314         start_time = time.time()
2315         formats = [f for f in formats if f.get('is_from_start')]
2316
2317         def refetch_manifest(format_id, delay):
2318             nonlocal formats, start_time, is_live
2319             if time.time() <= start_time + delay:
2320                 return
2321
2322             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2323             video_details = traverse_obj(
2324                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2325             microformats = traverse_obj(
2326                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2327                 expected_type=dict, default=[])
2328             _, is_live, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2329             start_time = time.time()
2330
2331         def mpd_feed(format_id, delay):
2332             """
2333             @returns (manifest_url, manifest_stream_number, is_live) or None
2334             """
2335             with lock:
2336                 refetch_manifest(format_id, delay)
2337
2338             f = next((f for f in formats if f['format_id'] == format_id), None)
2339             if not f:
2340                 if not is_live:
2341                     self.to_screen(f'{video_id}: Video is no longer live')
2342                 else:
2343                     self.report_warning(
2344                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2345                 return None
2346             return f['manifest_url'], f['manifest_stream_number'], is_live
2347
2348         for f in formats:
2349             f['is_live'] = True
2350             f['protocol'] = 'http_dash_segments_generator'
2351             f['fragments'] = functools.partial(
2352                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2353
2354     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2355         FETCH_SPAN, MAX_DURATION = 5, 432000
2356
2357         mpd_url, stream_number, is_live = None, None, True
2358
2359         begin_index = 0
2360         download_start_time = ctx.get('start') or time.time()
2361
2362         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2363         if lack_early_segments:
2364             self.report_warning(bug_reports_message(
2365                 'Starting download from the last 120 hours of the live stream since '
2366                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2367             lack_early_segments = True
2368
2369         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2370         fragments, fragment_base_url = None, None
2371
2372         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2373             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2374             # Obtain from MPD's maximum seq value
2375             old_mpd_url = mpd_url
2376             last_error = ctx.pop('last_error', None)
2377             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2378             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2379                                                or (mpd_url, stream_number, False))
2380             if not refresh_sequence:
2381                 if expire_fast and not is_live:
2382                     return False, last_seq
2383                 elif old_mpd_url == mpd_url:
2384                     return True, last_seq
2385             try:
2386                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2387                     mpd_url, None, note=False, errnote=False, fatal=False)
2388             except ExtractorError:
2389                 fmts = None
2390             if not fmts:
2391                 no_fragment_score += 2
2392                 return False, last_seq
2393             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2394             fragments = fmt_info['fragments']
2395             fragment_base_url = fmt_info['fragment_base_url']
2396             assert fragment_base_url
2397
2398             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2399             return True, _last_seq
2400
2401         while is_live:
2402             fetch_time = time.time()
2403             if no_fragment_score > 30:
2404                 return
2405             if last_segment_url:
2406                 # Obtain from "X-Head-Seqnum" header value from each segment
2407                 try:
2408                     urlh = self._request_webpage(
2409                         last_segment_url, None, note=False, errnote=False, fatal=False)
2410                 except ExtractorError:
2411                     urlh = None
2412                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2413                 if last_seq is None:
2414                     no_fragment_score += 2
2415                     last_segment_url = None
2416                     continue
2417             else:
2418                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2419                 no_fragment_score += 2
2420                 if not should_continue:
2421                     continue
2422
2423             if known_idx > last_seq:
2424                 last_segment_url = None
2425                 continue
2426
2427             last_seq += 1
2428
2429             if begin_index < 0 and known_idx < 0:
2430                 # skip from the start when it's negative value
2431                 known_idx = last_seq + begin_index
2432             if lack_early_segments:
2433                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2434             try:
2435                 for idx in range(known_idx, last_seq):
2436                     # do not update sequence here or you'll get skipped some part of it
2437                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2438                     if not should_continue:
2439                         known_idx = idx - 1
2440                         raise ExtractorError('breaking out of outer loop')
2441                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2442                     yield {
2443                         'url': last_segment_url,
2444                         'fragment_count': last_seq,
2445                     }
2446                 if known_idx == last_seq:
2447                     no_fragment_score += 5
2448                 else:
2449                     no_fragment_score = 0
2450                 known_idx = last_seq
2451             except ExtractorError:
2452                 continue
2453
2454             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2455
2456     def _extract_player_url(self, *ytcfgs, webpage=None):
2457         player_url = traverse_obj(
2458             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2459             get_all=False, expected_type=str)
2460         if not player_url:
2461             return
2462         return urljoin('https://www.youtube.com', player_url)
2463
2464     def _download_player_url(self, video_id, fatal=False):
2465         res = self._download_webpage(
2466             'https://www.youtube.com/iframe_api',
2467             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2468         if res:
2469             player_version = self._search_regex(
2470                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2471             if player_version:
2472                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2473
2474     def _signature_cache_id(self, example_sig):
2475         """ Return a string representation of a signature """
2476         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2477
2478     @classmethod
2479     def _extract_player_info(cls, player_url):
2480         for player_re in cls._PLAYER_INFO_RE:
2481             id_m = re.search(player_re, player_url)
2482             if id_m:
2483                 break
2484         else:
2485             raise ExtractorError('Cannot identify player %r' % player_url)
2486         return id_m.group('id')
2487
2488     def _load_player(self, video_id, player_url, fatal=True):
2489         player_id = self._extract_player_info(player_url)
2490         if player_id not in self._code_cache:
2491             code = self._download_webpage(
2492                 player_url, video_id, fatal=fatal,
2493                 note='Downloading player ' + player_id,
2494                 errnote='Download of %s failed' % player_url)
2495             if code:
2496                 self._code_cache[player_id] = code
2497         return self._code_cache.get(player_id)
2498
2499     def _extract_signature_function(self, video_id, player_url, example_sig):
2500         player_id = self._extract_player_info(player_url)
2501
2502         # Read from filesystem cache
2503         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2504         assert os.path.basename(func_id) == func_id
2505
2506         self.write_debug(f'Extracting signature function {func_id}')
2507         cache_spec = self.cache.load('youtube-sigfuncs', func_id)
2508         if cache_spec is not None:
2509             return lambda s: ''.join(s[i] for i in cache_spec)
2510
2511         code = self._load_player(video_id, player_url)
2512         if code:
2513             res = self._parse_sig_js(code)
2514
2515             test_string = ''.join(map(chr, range(len(example_sig))))
2516             cache_res = res(test_string)
2517             cache_spec = [ord(c) for c in cache_res]
2518
2519             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2520             return res
2521
2522     def _print_sig_code(self, func, example_sig):
2523         if not self.get_param('youtube_print_sig_code'):
2524             return
2525
2526         def gen_sig_code(idxs):
2527             def _genslice(start, end, step):
2528                 starts = '' if start == 0 else str(start)
2529                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2530                 steps = '' if step == 1 else (':%d' % step)
2531                 return f's[{starts}{ends}{steps}]'
2532
2533             step = None
2534             # Quelch pyflakes warnings - start will be set when step is set
2535             start = '(Never used)'
2536             for i, prev in zip(idxs[1:], idxs[:-1]):
2537                 if step is not None:
2538                     if i - prev == step:
2539                         continue
2540                     yield _genslice(start, prev, step)
2541                     step = None
2542                     continue
2543                 if i - prev in [-1, 1]:
2544                     step = i - prev
2545                     start = prev
2546                     continue
2547                 else:
2548                     yield 's[%d]' % prev
2549             if step is None:
2550                 yield 's[%d]' % i
2551             else:
2552                 yield _genslice(start, i, step)
2553
2554         test_string = ''.join(map(chr, range(len(example_sig))))
2555         cache_res = func(test_string)
2556         cache_spec = [ord(c) for c in cache_res]
2557         expr_code = ' + '.join(gen_sig_code(cache_spec))
2558         signature_id_tuple = '(%s)' % (
2559             ', '.join(str(len(p)) for p in example_sig.split('.')))
2560         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2561                 '    return %s\n') % (signature_id_tuple, expr_code)
2562         self.to_screen('Extracted signature function:\n' + code)
2563
2564     def _parse_sig_js(self, jscode):
2565         funcname = self._search_regex(
2566             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2567              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2568              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2569              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2570              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2571              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2572              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2573              # Obsolete patterns
2574              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2575              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2576              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2577              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2578              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2579              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2580              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2581              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2582             jscode, 'Initial JS player signature function name', group='sig')
2583
2584         jsi = JSInterpreter(jscode)
2585         initial_function = jsi.extract_function(funcname)
2586         return lambda s: initial_function([s])
2587
2588     def _decrypt_signature(self, s, video_id, player_url):
2589         """Turn the encrypted s field into a working signature"""
2590         try:
2591             player_id = (player_url, self._signature_cache_id(s))
2592             if player_id not in self._player_cache:
2593                 func = self._extract_signature_function(video_id, player_url, s)
2594                 self._player_cache[player_id] = func
2595             func = self._player_cache[player_id]
2596             self._print_sig_code(func, s)
2597             return func(s)
2598         except Exception as e:
2599             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2600
2601     def _decrypt_nsig(self, s, video_id, player_url):
2602         """Turn the encrypted n field into a working signature"""
2603         if player_url is None:
2604             raise ExtractorError('Cannot decrypt nsig without player_url')
2605         player_url = urljoin('https://www.youtube.com', player_url)
2606
2607         sig_id = ('nsig_value', s)
2608         if sig_id in self._player_cache:
2609             return self._player_cache[sig_id]
2610
2611         try:
2612             player_id = ('nsig', player_url)
2613             if player_id not in self._player_cache:
2614                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2615             func = self._player_cache[player_id]
2616             self._player_cache[sig_id] = func(s)
2617             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2618             return self._player_cache[sig_id]
2619         except Exception as e:
2620             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2621
2622     def _extract_n_function_name(self, jscode):
2623         nfunc, idx = self._search_regex(
2624             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2625             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2626         if not idx:
2627             return nfunc
2628         return json.loads(js_to_json(self._search_regex(
2629             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2630             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2631
2632     def _extract_n_function(self, video_id, player_url):
2633         player_id = self._extract_player_info(player_url)
2634         func_code = self.cache.load('youtube-nsig', player_id)
2635
2636         if func_code:
2637             jsi = JSInterpreter(func_code)
2638         else:
2639             jscode = self._load_player(video_id, player_url)
2640             funcname = self._extract_n_function_name(jscode)
2641             jsi = JSInterpreter(jscode)
2642             func_code = jsi.extract_function_code(funcname)
2643             self.cache.store('youtube-nsig', player_id, func_code)
2644
2645         if self.get_param('youtube_print_sig_code'):
2646             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2647
2648         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2649
2650     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2651         """
2652         Extract signatureTimestamp (sts)
2653         Required to tell API what sig/player version is in use.
2654         """
2655         sts = None
2656         if isinstance(ytcfg, dict):
2657             sts = int_or_none(ytcfg.get('STS'))
2658
2659         if not sts:
2660             # Attempt to extract from player
2661             if player_url is None:
2662                 error_msg = 'Cannot extract signature timestamp without player_url.'
2663                 if fatal:
2664                     raise ExtractorError(error_msg)
2665                 self.report_warning(error_msg)
2666                 return
2667             code = self._load_player(video_id, player_url, fatal=fatal)
2668             if code:
2669                 sts = int_or_none(self._search_regex(
2670                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2671                     'JS player signature timestamp', group='sts', fatal=fatal))
2672         return sts
2673
2674     def _mark_watched(self, video_id, player_responses):
2675         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2676             label = 'fully ' if is_full else ''
2677             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2678                             expected_type=url_or_none)
2679             if not url:
2680                 self.report_warning(f'Unable to mark {label}watched')
2681                 return
2682             parsed_url = urllib.parse.urlparse(url)
2683             qs = urllib.parse.parse_qs(parsed_url.query)
2684
2685             # cpn generation algorithm is reverse engineered from base.js.
2686             # In fact it works even with dummy cpn.
2687             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2688             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2689
2690             # # more consistent results setting it to right before the end
2691             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2692
2693             qs.update({
2694                 'ver': ['2'],
2695                 'cpn': [cpn],
2696                 'cmt': video_length,
2697                 'el': 'detailpage',  # otherwise defaults to "shorts"
2698             })
2699
2700             if is_full:
2701                 # these seem to mark watchtime "history" in the real world
2702                 # they're required, so send in a single value
2703                 qs.update({
2704                     'st': video_length,
2705                     'et': video_length,
2706                 })
2707
2708             url = urllib.parse.urlunparse(
2709                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2710
2711             self._download_webpage(
2712                 url, video_id, f'Marking {label}watched',
2713                 'Unable to mark watched', fatal=False)
2714
2715     @classmethod
2716     def _extract_from_webpage(cls, url, webpage):
2717         # Invidious Instances
2718         # https://github.com/yt-dlp/yt-dlp/issues/195
2719         # https://github.com/iv-org/invidious/pull/1730
2720         mobj = re.search(
2721             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
2722             webpage)
2723         if mobj:
2724             yield cls.url_result(mobj.group('url'), cls)
2725             raise cls.StopExtraction()
2726
2727         yield from super()._extract_from_webpage(url, webpage)
2728
2729         # lazyYT YouTube embed
2730         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
2731             yield cls.url_result(unescapeHTML(id_), cls, id_)
2732
2733         # Wordpress "YouTube Video Importer" plugin
2734         for m in re.findall(r'''(?x)<div[^>]+
2735                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2736                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
2737             yield cls.url_result(m[-1], cls, m[-1])
2738
2739     @classmethod
2740     def extract_id(cls, url):
2741         video_id = cls.get_temp_id(url)
2742         if not video_id:
2743             raise ExtractorError(f'Invalid URL: {url}')
2744         return video_id
2745
2746     def _extract_chapters_from_json(self, data, duration):
2747         chapter_list = traverse_obj(
2748             data, (
2749                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2750                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2751             ), expected_type=list)
2752
2753         return self._extract_chapters(
2754             chapter_list,
2755             chapter_time=lambda chapter: float_or_none(
2756                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2757             chapter_title=lambda chapter: traverse_obj(
2758                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2759             duration=duration)
2760
2761     def _extract_chapters_from_engagement_panel(self, data, duration):
2762         content_list = traverse_obj(
2763             data,
2764             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2765             expected_type=list, default=[])
2766         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2767         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2768
2769         return next(filter(None, (
2770             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2771                                    chapter_time, chapter_title, duration)
2772             for contents in content_list)), [])
2773
2774     def _extract_chapters_from_description(self, description, duration):
2775         return self._extract_chapters(
2776             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
2777             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
2778             duration=duration, strict=False)
2779
2780     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
2781         if not duration:
2782             return
2783         chapter_list = [{
2784             'start_time': chapter_time(chapter),
2785             'title': chapter_title(chapter),
2786         } for chapter in chapter_list or []]
2787         if not strict:
2788             chapter_list.sort(key=lambda c: c['start_time'] or 0)
2789
2790         chapters = [{'start_time': 0}]
2791         for idx, chapter in enumerate(chapter_list):
2792             if chapter['start_time'] is None:
2793                 self.report_warning(f'Incomplete chapter {idx}')
2794             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
2795                 chapters.append(chapter)
2796             else:
2797                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
2798         return chapters[1:]
2799
2800     def _extract_comment(self, comment_renderer, parent=None):
2801         comment_id = comment_renderer.get('commentId')
2802         if not comment_id:
2803             return
2804
2805         text = self._get_text(comment_renderer, 'contentText')
2806
2807         # note: timestamp is an estimate calculated from the current time and time_text
2808         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2809         author = self._get_text(comment_renderer, 'authorText')
2810         author_id = try_get(comment_renderer,
2811                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
2812
2813         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2814                                                        lambda x: x['likeCount']), str)) or 0
2815         author_thumbnail = try_get(comment_renderer,
2816                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
2817
2818         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2819         is_favorited = 'creatorHeart' in (try_get(
2820             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2821         return {
2822             'id': comment_id,
2823             'text': text,
2824             'timestamp': timestamp,
2825             'time_text': time_text,
2826             'like_count': votes,
2827             'is_favorited': is_favorited,
2828             'author': author,
2829             'author_id': author_id,
2830             'author_thumbnail': author_thumbnail,
2831             'author_is_uploader': author_is_uploader,
2832             'parent': parent or 'root'
2833         }
2834
2835     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2836
2837         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2838
2839         def extract_header(contents):
2840             _continuation = None
2841             for content in contents:
2842                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2843                 expected_comment_count = self._get_count(
2844                     comments_header_renderer, 'countText', 'commentsCount')
2845
2846                 if expected_comment_count:
2847                     tracker['est_total'] = expected_comment_count
2848                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2849                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2850
2851                 sort_menu_item = try_get(
2852                     comments_header_renderer,
2853                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2854                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2855
2856                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2857                 if not _continuation:
2858                     continue
2859
2860                 sort_text = str_or_none(sort_menu_item.get('title'))
2861                 if not sort_text:
2862                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2863                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2864                 break
2865             return _continuation
2866
2867         def extract_thread(contents):
2868             if not parent:
2869                 tracker['current_page_thread'] = 0
2870             for content in contents:
2871                 if not parent and tracker['total_parent_comments'] >= max_parents:
2872                     yield
2873                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2874                 comment_renderer = get_first(
2875                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2876                     expected_type=dict, default={})
2877
2878                 comment = self._extract_comment(comment_renderer, parent)
2879                 if not comment:
2880                     continue
2881
2882                 tracker['running_total'] += 1
2883                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2884                 yield comment
2885
2886                 # Attempt to get the replies
2887                 comment_replies_renderer = try_get(
2888                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2889
2890                 if comment_replies_renderer:
2891                     tracker['current_page_thread'] += 1
2892                     comment_entries_iter = self._comment_entries(
2893                         comment_replies_renderer, ytcfg, video_id,
2894                         parent=comment.get('id'), tracker=tracker)
2895                     yield from itertools.islice(comment_entries_iter, min(
2896                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2897
2898         # Keeps track of counts across recursive calls
2899         if not tracker:
2900             tracker = dict(
2901                 running_total=0,
2902                 est_total=0,
2903                 current_page_thread=0,
2904                 total_parent_comments=0,
2905                 total_reply_comments=0)
2906
2907         # TODO: Deprecated
2908         # YouTube comments have a max depth of 2
2909         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2910         if max_depth:
2911             self._downloader.deprecation_warning(
2912                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2913         if max_depth == 1 and parent:
2914             return
2915
2916         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2917             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2918
2919         continuation = self._extract_continuation(root_continuation_data)
2920
2921         response = None
2922         is_forced_continuation = False
2923         is_first_continuation = parent is None
2924         if is_first_continuation and not continuation:
2925             # Sometimes you can get comments by generating the continuation yourself,
2926             # even if YouTube initially reports them being disabled - e.g. stories comments.
2927             # Note: if the comment section is actually disabled, YouTube may return a response with
2928             # required check_get_keys missing. So we will disable that check initially in this case.
2929             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2930             is_forced_continuation = True
2931
2932         for page_num in itertools.count(0):
2933             if not continuation:
2934                 break
2935             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2936             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2937             if page_num == 0:
2938                 if is_first_continuation:
2939                     note_prefix = 'Downloading comment section API JSON'
2940                 else:
2941                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2942                         tracker['current_page_thread'], comment_prog_str)
2943             else:
2944                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2945                     '       ' if parent else '', ' replies' if parent else '',
2946                     page_num, comment_prog_str)
2947
2948             response = self._extract_response(
2949                 item_id=None, query=continuation,
2950                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2951                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2952             is_forced_continuation = False
2953             continuation_contents = traverse_obj(
2954                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2955
2956             continuation = None
2957             for continuation_section in continuation_contents:
2958                 continuation_items = traverse_obj(
2959                     continuation_section,
2960                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2961                     get_all=False, expected_type=list) or []
2962                 if is_first_continuation:
2963                     continuation = extract_header(continuation_items)
2964                     is_first_continuation = False
2965                     if continuation:
2966                         break
2967                     continue
2968
2969                 for entry in extract_thread(continuation_items):
2970                     if not entry:
2971                         return
2972                     yield entry
2973                 continuation = self._extract_continuation({'contents': continuation_items})
2974                 if continuation:
2975                     break
2976
2977         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2978         if message and not parent and tracker['running_total'] == 0:
2979             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2980
2981     @staticmethod
2982     def _generate_comment_continuation(video_id):
2983         """
2984         Generates initial comment section continuation token from given video id
2985         """
2986         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2987         return base64.b64encode(token.encode()).decode()
2988
2989     def _get_comments(self, ytcfg, video_id, contents, webpage):
2990         """Entry for comment extraction"""
2991         def _real_comment_extract(contents):
2992             renderer = next((
2993                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2994                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2995             yield from self._comment_entries(renderer, ytcfg, video_id)
2996
2997         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2998         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2999
3000     @staticmethod
3001     def _get_checkok_params():
3002         return {'contentCheckOk': True, 'racyCheckOk': True}
3003
3004     @classmethod
3005     def _generate_player_context(cls, sts=None):
3006         context = {
3007             'html5Preference': 'HTML5_PREF_WANTS',
3008         }
3009         if sts is not None:
3010             context['signatureTimestamp'] = sts
3011         return {
3012             'playbackContext': {
3013                 'contentPlaybackContext': context
3014             },
3015             **cls._get_checkok_params()
3016         }
3017
3018     @staticmethod
3019     def _is_agegated(player_response):
3020         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3021             return True
3022
3023         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3024         AGE_GATE_REASONS = (
3025             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3026             'age_verification_required', 'age_check_required',  # status
3027         )
3028         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3029
3030     @staticmethod
3031     def _is_unplayable(player_response):
3032         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3033
3034     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
3035
3036         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3037         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3038         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3039         headers = self.generate_api_headers(
3040             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3041
3042         yt_query = {
3043             'videoId': video_id,
3044             'params': '8AEB'  # enable stories
3045         }
3046         yt_query.update(self._generate_player_context(sts))
3047         return self._extract_response(
3048             item_id=video_id, ep='player', query=yt_query,
3049             ytcfg=player_ytcfg, headers=headers, fatal=True,
3050             default_client=client,
3051             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3052         ) or None
3053
3054     def _get_requested_clients(self, url, smuggled_data):
3055         requested_clients = []
3056         default = ['android', 'web']
3057         allowed_clients = sorted(
3058             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3059             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3060         for client in self._configuration_arg('player_client'):
3061             if client in allowed_clients:
3062                 requested_clients.append(client)
3063             elif client == 'default':
3064                 requested_clients.extend(default)
3065             elif client == 'all':
3066                 requested_clients.extend(allowed_clients)
3067             else:
3068                 self.report_warning(f'Skipping unsupported client {client}')
3069         if not requested_clients:
3070             requested_clients = default
3071
3072         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3073             requested_clients.extend(
3074                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3075
3076         return orderedSet(requested_clients)
3077
3078     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3079         initial_pr = None
3080         if webpage:
3081             initial_pr = self._search_json(
3082                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3083
3084         all_clients = set(clients)
3085         clients = clients[::-1]
3086         prs = []
3087
3088         def append_client(*client_names):
3089             """ Append the first client name that exists but not already used """
3090             for client_name in client_names:
3091                 actual_client = _split_innertube_client(client_name)[0]
3092                 if actual_client in INNERTUBE_CLIENTS:
3093                     if actual_client not in all_clients:
3094                         clients.append(client_name)
3095                         all_clients.add(actual_client)
3096                         return
3097
3098         # Android player_response does not have microFormats which are needed for
3099         # extraction of some data. So we return the initial_pr with formats
3100         # stripped out even if not requested by the user
3101         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3102         if initial_pr:
3103             pr = dict(initial_pr)
3104             pr['streamingData'] = None
3105             prs.append(pr)
3106
3107         last_error = None
3108         tried_iframe_fallback = False
3109         player_url = None
3110         while clients:
3111             client, base_client, variant = _split_innertube_client(clients.pop())
3112             player_ytcfg = master_ytcfg if client == 'web' else {}
3113             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3114                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3115
3116             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3117             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3118             if 'js' in self._configuration_arg('player_skip'):
3119                 require_js_player = False
3120                 player_url = None
3121
3122             if not player_url and not tried_iframe_fallback and require_js_player:
3123                 player_url = self._download_player_url(video_id)
3124                 tried_iframe_fallback = True
3125
3126             try:
3127                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3128                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3129             except ExtractorError as e:
3130                 if last_error:
3131                     self.report_warning(last_error)
3132                 last_error = e
3133                 continue
3134
3135             if pr:
3136                 # YouTube may return a different video player response than expected.
3137                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3138                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3139                 if pr_video_id and pr_video_id != video_id:
3140                     self.report_warning(
3141                         f'{client} client returned a player response for "{pr_video_id}" instead of "{video_id}"' + bug_reports_message())
3142                 else:
3143                     prs.append(pr)
3144
3145             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3146             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3147                 append_client(f'{base_client}_creator')
3148             elif self._is_agegated(pr):
3149                 if variant == 'tv_embedded':
3150                     append_client(f'{base_client}_embedded')
3151                 elif not variant:
3152                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3153
3154         if last_error:
3155             if not len(prs):
3156                 raise last_error
3157             self.report_warning(last_error)
3158         return prs, player_url
3159
3160     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, is_live, duration):
3161         itags, stream_ids = {}, []
3162         itag_qualities, res_qualities = {}, {}
3163         q = qualities([
3164             # Normally tiny is the smallest video-only formats. But
3165             # audio-only formats with unknown quality may get tagged as tiny
3166             'tiny',
3167             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3168             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3169         ])
3170         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3171
3172         for fmt in streaming_formats:
3173             if fmt.get('targetDurationSec'):
3174                 continue
3175
3176             itag = str_or_none(fmt.get('itag'))
3177             audio_track = fmt.get('audioTrack') or {}
3178             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3179             if stream_id in stream_ids:
3180                 continue
3181
3182             quality = fmt.get('quality')
3183             height = int_or_none(fmt.get('height'))
3184             if quality == 'tiny' or not quality:
3185                 quality = fmt.get('audioQuality', '').lower() or quality
3186             # The 3gp format (17) in android client has a quality of "small",
3187             # but is actually worse than other formats
3188             if itag == '17':
3189                 quality = 'tiny'
3190             if quality:
3191                 if itag:
3192                     itag_qualities[itag] = quality
3193                 if height:
3194                     res_qualities[height] = quality
3195             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3196             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3197             # number of fragment that would subsequently requested with (`&sq=N`)
3198             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3199                 continue
3200
3201             fmt_url = fmt.get('url')
3202             if not fmt_url:
3203                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3204                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3205                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3206                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3207                     continue
3208                 try:
3209                     fmt_url += '&%s=%s' % (
3210                         traverse_obj(sc, ('sp', -1)) or 'signature',
3211                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3212                     )
3213                 except ExtractorError as e:
3214                     self.report_warning('Signature extraction failed: Some formats may be missing', only_once=True)
3215                     self.write_debug(e, only_once=True)
3216                     continue
3217
3218             query = parse_qs(fmt_url)
3219             throttled = False
3220             if query.get('n'):
3221                 try:
3222                     fmt_url = update_url_query(fmt_url, {
3223                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3224                 except ExtractorError as e:
3225                     self.report_warning(
3226                         'nsig extraction failed: You may experience throttling for some formats\n'
3227                         f'n = {query["n"][0]} ; player = {player_url}', only_once=True)
3228                     self.write_debug(e, only_once=True)
3229                     throttled = True
3230
3231             if itag:
3232                 itags[itag] = 'https'
3233                 stream_ids.append(stream_id)
3234
3235             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3236             language_preference = (
3237                 10 if audio_track.get('audioIsDefault') and 10
3238                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3239                 else -1)
3240             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3241             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3242             # Make sure to avoid false positives with small duration differences.
3243             # Eg: __2ABJjxzNo, ySuUZEjARPY
3244             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3245             if is_damaged:
3246                 self.report_warning(
3247                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3248             dct = {
3249                 'asr': int_or_none(fmt.get('audioSampleRate')),
3250                 'filesize': int_or_none(fmt.get('contentLength')),
3251                 'format_id': itag,
3252                 'format_note': join_nonempty(
3253                     '%s%s' % (audio_track.get('displayName') or '',
3254                               ' (default)' if language_preference > 0 else ''),
3255                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3256                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3257                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3258                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3259                 'fps': int_or_none(fmt.get('fps')) or None,
3260                 'height': height,
3261                 'quality': q(quality),
3262                 'has_drm': bool(fmt.get('drmFamilies')),
3263                 'tbr': tbr,
3264                 'url': fmt_url,
3265                 'width': int_or_none(fmt.get('width')),
3266                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3267                                           'desc' if language_preference < -1 else ''),
3268                 'language_preference': language_preference,
3269                 # Strictly de-prioritize damaged and 3gp formats
3270                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3271             }
3272             mime_mobj = re.match(
3273                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3274             if mime_mobj:
3275                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3276                 dct.update(parse_codecs(mime_mobj.group(2)))
3277             no_audio = dct.get('acodec') == 'none'
3278             no_video = dct.get('vcodec') == 'none'
3279             if no_audio:
3280                 dct['vbr'] = tbr
3281             if no_video:
3282                 dct['abr'] = tbr
3283             if no_audio or no_video:
3284                 dct['downloader_options'] = {
3285                     # Youtube throttles chunks >~10M
3286                     'http_chunk_size': 10485760,
3287                 }
3288                 if dct.get('ext'):
3289                     dct['container'] = dct['ext'] + '_dash'
3290             yield dct
3291
3292         live_from_start = is_live and self.get_param('live_from_start')
3293         skip_manifests = self._configuration_arg('skip')
3294         if not self.get_param('youtube_include_hls_manifest', True):
3295             skip_manifests.append('hls')
3296         if not self.get_param('youtube_include_dash_manifest', True):
3297             skip_manifests.append('dash')
3298         get_dash = 'dash' not in skip_manifests and (
3299             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3300         get_hls = not live_from_start and 'hls' not in skip_manifests
3301
3302         def process_manifest_format(f, proto, itag):
3303             if itag in itags:
3304                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3305                     return False
3306                 itag = f'{itag}-{proto}'
3307             if itag:
3308                 f['format_id'] = itag
3309                 itags[itag] = proto
3310
3311             f['quality'] = next((
3312                 q(qdict[val])
3313                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3314                 if val in qdict), -1)
3315             return True
3316
3317         subtitles = {}
3318         for sd in streaming_data:
3319             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3320             if hls_manifest_url:
3321                 fmts, subs = self._extract_m3u8_formats_and_subtitles(hls_manifest_url, video_id, 'mp4', fatal=False, live=is_live)
3322                 subtitles = self._merge_subtitles(subs, subtitles)
3323                 for f in fmts:
3324                     if process_manifest_format(f, 'hls', self._search_regex(
3325                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3326                         yield f
3327
3328             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3329             if dash_manifest_url:
3330                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3331                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3332                 for f in formats:
3333                     if process_manifest_format(f, 'dash', f['format_id']):
3334                         f['filesize'] = int_or_none(self._search_regex(
3335                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3336                         if live_from_start:
3337                             f['is_from_start'] = True
3338
3339                         yield f
3340         yield subtitles
3341
3342     def _extract_storyboard(self, player_responses, duration):
3343         spec = get_first(
3344             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3345         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3346         if not base_url:
3347             return
3348         L = len(spec) - 1
3349         for i, args in enumerate(spec):
3350             args = args.split('#')
3351             counts = list(map(int_or_none, args[:5]))
3352             if len(args) != 8 or not all(counts):
3353                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3354                 continue
3355             width, height, frame_count, cols, rows = counts
3356             N, sigh = args[6:]
3357
3358             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3359             fragment_count = frame_count / (cols * rows)
3360             fragment_duration = duration / fragment_count
3361             yield {
3362                 'format_id': f'sb{i}',
3363                 'format_note': 'storyboard',
3364                 'ext': 'mhtml',
3365                 'protocol': 'mhtml',
3366                 'acodec': 'none',
3367                 'vcodec': 'none',
3368                 'url': url,
3369                 'width': width,
3370                 'height': height,
3371                 'fps': frame_count / duration,
3372                 'rows': rows,
3373                 'columns': cols,
3374                 'fragments': [{
3375                     'url': url.replace('$M', str(j)),
3376                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3377                 } for j in range(math.ceil(fragment_count))],
3378             }
3379
3380     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3381         webpage = None
3382         if 'webpage' not in self._configuration_arg('player_skip'):
3383             webpage = self._download_webpage(
3384                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3385
3386         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3387
3388         player_responses, player_url = self._extract_player_responses(
3389             self._get_requested_clients(url, smuggled_data),
3390             video_id, webpage, master_ytcfg)
3391
3392         return webpage, master_ytcfg, player_responses, player_url
3393
3394     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3395         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3396         is_live = get_first(video_details, 'isLive')
3397         if is_live is None:
3398             is_live = get_first(live_broadcast_details, 'isLiveNow')
3399
3400         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3401         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, is_live, duration)
3402
3403         return live_broadcast_details, is_live, streaming_data, formats, subtitles
3404
3405     def _real_extract(self, url):
3406         url, smuggled_data = unsmuggle_url(url, {})
3407         video_id = self._match_id(url)
3408
3409         base_url = self.http_scheme() + '//www.youtube.com/'
3410         webpage_url = base_url + 'watch?v=' + video_id
3411
3412         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3413
3414         playability_statuses = traverse_obj(
3415             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3416
3417         trailer_video_id = get_first(
3418             playability_statuses,
3419             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3420             expected_type=str)
3421         if trailer_video_id:
3422             return self.url_result(
3423                 trailer_video_id, self.ie_key(), trailer_video_id)
3424
3425         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3426                        if webpage else (lambda x: None))
3427
3428         video_details = traverse_obj(
3429             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3430         microformats = traverse_obj(
3431             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3432             expected_type=dict, default=[])
3433         video_title = (
3434             get_first(video_details, 'title')
3435             or self._get_text(microformats, (..., 'title'))
3436             or search_meta(['og:title', 'twitter:title', 'title']))
3437         video_description = get_first(video_details, 'shortDescription')
3438
3439         multifeed_metadata_list = get_first(
3440             player_responses,
3441             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3442             expected_type=str)
3443         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3444             if self.get_param('noplaylist'):
3445                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3446             else:
3447                 entries = []
3448                 feed_ids = []
3449                 for feed in multifeed_metadata_list.split(','):
3450                     # Unquote should take place before split on comma (,) since textual
3451                     # fields may contain comma as well (see
3452                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3453                     feed_data = urllib.parse.parse_qs(
3454                         urllib.parse.unquote_plus(feed))
3455
3456                     def feed_entry(name):
3457                         return try_get(
3458                             feed_data, lambda x: x[name][0], str)
3459
3460                     feed_id = feed_entry('id')
3461                     if not feed_id:
3462                         continue
3463                     feed_title = feed_entry('title')
3464                     title = video_title
3465                     if feed_title:
3466                         title += ' (%s)' % feed_title
3467                     entries.append({
3468                         '_type': 'url_transparent',
3469                         'ie_key': 'Youtube',
3470                         'url': smuggle_url(
3471                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3472                             {'force_singlefeed': True}),
3473                         'title': title,
3474                     })
3475                     feed_ids.append(feed_id)
3476                 self.to_screen(
3477                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3478                     % (', '.join(feed_ids), video_id))
3479                 return self.playlist_result(
3480                     entries, video_id, video_title, video_description)
3481
3482         duration = int_or_none(
3483             get_first(video_details, 'lengthSeconds')
3484             or get_first(microformats, 'lengthSeconds')
3485             or parse_duration(search_meta('duration'))) or None
3486
3487         live_broadcast_details, is_live, streaming_data, formats, automatic_captions = \
3488             self._list_formats(video_id, microformats, video_details, player_responses, player_url)
3489
3490         if not formats:
3491             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3492                 self.report_drm(video_id)
3493             pemr = get_first(
3494                 playability_statuses,
3495                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3496             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3497             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3498             if subreason:
3499                 if subreason == 'The uploader has not made this video available in your country.':
3500                     countries = get_first(microformats, 'availableCountries')
3501                     if not countries:
3502                         regions_allowed = search_meta('regionsAllowed')
3503                         countries = regions_allowed.split(',') if regions_allowed else None
3504                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3505                 reason += f'. {subreason}'
3506             if reason:
3507                 self.raise_no_formats(reason, expected=True)
3508
3509         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3510         if not keywords and webpage:
3511             keywords = [
3512                 unescapeHTML(m.group('content'))
3513                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3514         for keyword in keywords:
3515             if keyword.startswith('yt:stretch='):
3516                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3517                 if mobj:
3518                     # NB: float is intentional for forcing float division
3519                     w, h = (float(v) for v in mobj.groups())
3520                     if w > 0 and h > 0:
3521                         ratio = w / h
3522                         for f in formats:
3523                             if f.get('vcodec') != 'none':
3524                                 f['stretched_ratio'] = ratio
3525                         break
3526         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3527         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3528         if thumbnail_url:
3529             thumbnails.append({
3530                 'url': thumbnail_url,
3531             })
3532         original_thumbnails = thumbnails.copy()
3533
3534         # The best resolution thumbnails sometimes does not appear in the webpage
3535         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3536         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3537         thumbnail_names = [
3538             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3539             # in resolution, these are not the custom thumbnail. So de-prioritize them
3540             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3541             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3542         ]
3543         n_thumbnail_names = len(thumbnail_names)
3544         thumbnails.extend({
3545             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3546                 video_id=video_id, name=name, ext=ext,
3547                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3548         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3549         for thumb in thumbnails:
3550             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3551             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3552         self._remove_duplicate_formats(thumbnails)
3553         self._downloader._sort_thumbnails(original_thumbnails)
3554
3555         category = get_first(microformats, 'category') or search_meta('genre')
3556         channel_id = str_or_none(
3557             get_first(video_details, 'channelId')
3558             or get_first(microformats, 'externalChannelId')
3559             or search_meta('channelId'))
3560         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3561
3562         live_content = get_first(video_details, 'isLiveContent')
3563         is_upcoming = get_first(video_details, 'isUpcoming')
3564         if is_live is None:
3565             if is_upcoming or live_content is False:
3566                 is_live = False
3567         if is_upcoming is None and (live_content or is_live):
3568             is_upcoming = False
3569         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3570         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3571         if not duration and live_end_time and live_start_time:
3572             duration = live_end_time - live_start_time
3573
3574         if is_live and self.get_param('live_from_start'):
3575             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3576
3577         formats.extend(self._extract_storyboard(player_responses, duration))
3578
3579         # source_preference is lower for throttled/potentially damaged formats
3580         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3581
3582         info = {
3583             'id': video_id,
3584             'title': video_title,
3585             'formats': formats,
3586             'thumbnails': thumbnails,
3587             # The best thumbnail that we are sure exists. Prevents unnecessary
3588             # URL checking if user don't care about getting the best possible thumbnail
3589             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3590             'description': video_description,
3591             'uploader': get_first(video_details, 'author'),
3592             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3593             'uploader_url': owner_profile_url,
3594             'channel_id': channel_id,
3595             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3596             'duration': duration,
3597             'view_count': int_or_none(
3598                 get_first((video_details, microformats), (..., 'viewCount'))
3599                 or search_meta('interactionCount')),
3600             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3601             'age_limit': 18 if (
3602                 get_first(microformats, 'isFamilySafe') is False
3603                 or search_meta('isFamilyFriendly') == 'false'
3604                 or search_meta('og:restrictions:age') == '18+') else 0,
3605             'webpage_url': webpage_url,
3606             'categories': [category] if category else None,
3607             'tags': keywords,
3608             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3609             'is_live': is_live,
3610             'was_live': (False if is_live or is_upcoming or live_content is False
3611                          else None if is_live is None or is_upcoming is None
3612                          else live_content),
3613             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3614             'release_timestamp': live_start_time,
3615         }
3616
3617         if get_first(video_details, 'isPostLiveDvr'):
3618             self.write_debug('Video is in Post-Live Manifestless mode')
3619             info['live_status'] = 'post_live'
3620             if (duration or 0) > 4 * 3600:
3621                 self.report_warning(
3622                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3623                     'This is a known issue and patches are welcome')
3624
3625         subtitles = {}
3626         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3627         if pctr:
3628             def get_lang_code(track):
3629                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3630                         or track.get('languageCode'))
3631
3632             # Converted into dicts to remove duplicates
3633             captions = {
3634                 get_lang_code(sub): sub
3635                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3636             translation_languages = {
3637                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3638                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3639
3640             def process_language(container, base_url, lang_code, sub_name, query):
3641                 lang_subs = container.setdefault(lang_code, [])
3642                 for fmt in self._SUBTITLE_FORMATS:
3643                     query.update({
3644                         'fmt': fmt,
3645                     })
3646                     lang_subs.append({
3647                         'ext': fmt,
3648                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3649                         'name': sub_name,
3650                     })
3651
3652             # NB: Constructing the full subtitle dictionary is slow
3653             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
3654                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
3655             for lang_code, caption_track in captions.items():
3656                 base_url = caption_track.get('baseUrl')
3657                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3658                 if not base_url:
3659                     continue
3660                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3661                 if caption_track.get('kind') != 'asr':
3662                     if not lang_code:
3663                         continue
3664                     process_language(
3665                         subtitles, base_url, lang_code, lang_name, {})
3666                     if not caption_track.get('isTranslatable'):
3667                         continue
3668                 for trans_code, trans_name in translation_languages.items():
3669                     if not trans_code:
3670                         continue
3671                     orig_trans_code = trans_code
3672                     if caption_track.get('kind') != 'asr':
3673                         if not get_translated_subs:
3674                             continue
3675                         trans_code += f'-{lang_code}'
3676                         trans_name += format_field(lang_name, None, ' from %s')
3677                     # Add an "-orig" label to the original language so that it can be distinguished.
3678                     # The subs are returned without "-orig" as well for compatibility
3679                     if lang_code == f'a-{orig_trans_code}':
3680                         process_language(
3681                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3682                     # Setting tlang=lang returns damaged subtitles.
3683                     process_language(automatic_captions, base_url, trans_code, trans_name,
3684                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3685
3686         info['automatic_captions'] = automatic_captions
3687         info['subtitles'] = subtitles
3688
3689         parsed_url = urllib.parse.urlparse(url)
3690         for component in [parsed_url.fragment, parsed_url.query]:
3691             query = urllib.parse.parse_qs(component)
3692             for k, v in query.items():
3693                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3694                     d_k += '_time'
3695                     if d_k not in info and k in s_ks:
3696                         info[d_k] = parse_duration(query[k][0])
3697
3698         # Youtube Music Auto-generated description
3699         if video_description:
3700             mobj = re.search(
3701                 r'''(?xs)
3702                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3703                     (?P<album>[^\n]+)
3704                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3705                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3706                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3707                     .+\nAuto-generated\ by\ YouTube\.\s*$
3708                 ''', video_description)
3709             if mobj:
3710                 release_year = mobj.group('release_year')
3711                 release_date = mobj.group('release_date')
3712                 if release_date:
3713                     release_date = release_date.replace('-', '')
3714                     if not release_year:
3715                         release_year = release_date[:4]
3716                 info.update({
3717                     'album': mobj.group('album'.strip()),
3718                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3719                     'track': mobj.group('track').strip(),
3720                     'release_date': release_date,
3721                     'release_year': int_or_none(release_year),
3722                 })
3723
3724         initial_data = None
3725         if webpage:
3726             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
3727         if not initial_data:
3728             query = {'videoId': video_id}
3729             query.update(self._get_checkok_params())
3730             initial_data = self._extract_response(
3731                 item_id=video_id, ep='next', fatal=False,
3732                 ytcfg=master_ytcfg, query=query,
3733                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3734                 note='Downloading initial data API JSON')
3735
3736         info['comment_count'] = traverse_obj(initial_data, (
3737             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
3738             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
3739         ), (
3740             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
3741             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
3742         ), expected_type=int_or_none, get_all=False)
3743
3744         try:  # This will error if there is no livechat
3745             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3746         except (KeyError, IndexError, TypeError):
3747             pass
3748         else:
3749             info.setdefault('subtitles', {})['live_chat'] = [{
3750                 # url is needed to set cookies
3751                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
3752                 'video_id': video_id,
3753                 'ext': 'json',
3754                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3755             }]
3756
3757         if initial_data:
3758             info['chapters'] = (
3759                 self._extract_chapters_from_json(initial_data, duration)
3760                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3761                 or self._extract_chapters_from_description(video_description, duration)
3762                 or None)
3763
3764         contents = traverse_obj(
3765             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3766             expected_type=list, default=[])
3767
3768         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3769         if vpir:
3770             stl = vpir.get('superTitleLink')
3771             if stl:
3772                 stl = self._get_text(stl)
3773                 if try_get(
3774                         vpir,
3775                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3776                     info['location'] = stl
3777                 else:
3778                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3779                     if mobj:
3780                         info.update({
3781                             'series': mobj.group(1),
3782                             'season_number': int(mobj.group(2)),
3783                             'episode_number': int(mobj.group(3)),
3784                         })
3785             for tlb in (try_get(
3786                     vpir,
3787                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3788                     list) or []):
3789                 tbr = tlb.get('toggleButtonRenderer') or {}
3790                 for getter, regex in [(
3791                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3792                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3793                             lambda x: x['accessibility'],
3794                             lambda x: x['accessibilityData']['accessibilityData'],
3795                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3796                     label = (try_get(tbr, getter, dict) or {}).get('label')
3797                     if label:
3798                         mobj = re.match(regex, label)
3799                         if mobj:
3800                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3801                             break
3802             sbr_tooltip = try_get(
3803                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3804             if sbr_tooltip:
3805                 like_count, dislike_count = sbr_tooltip.split(' / ')
3806                 info.update({
3807                     'like_count': str_to_int(like_count),
3808                     'dislike_count': str_to_int(dislike_count),
3809                 })
3810         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3811         if vsir:
3812             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3813             info.update({
3814                 'channel': self._get_text(vor, 'title'),
3815                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3816
3817             rows = try_get(
3818                 vsir,
3819                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3820                 list) or []
3821             multiple_songs = False
3822             for row in rows:
3823                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3824                     multiple_songs = True
3825                     break
3826             for row in rows:
3827                 mrr = row.get('metadataRowRenderer') or {}
3828                 mrr_title = mrr.get('title')
3829                 if not mrr_title:
3830                     continue
3831                 mrr_title = self._get_text(mrr, 'title')
3832                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3833                 if mrr_title == 'License':
3834                     info['license'] = mrr_contents_text
3835                 elif not multiple_songs:
3836                     if mrr_title == 'Album':
3837                         info['album'] = mrr_contents_text
3838                     elif mrr_title == 'Artist':
3839                         info['artist'] = mrr_contents_text
3840                     elif mrr_title == 'Song':
3841                         info['track'] = mrr_contents_text
3842
3843         fallbacks = {
3844             'channel': 'uploader',
3845             'channel_id': 'uploader_id',
3846             'channel_url': 'uploader_url',
3847         }
3848
3849         # The upload date for scheduled, live and past live streams / premieres in microformats
3850         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3851         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3852         upload_date = (
3853             unified_strdate(get_first(microformats, 'uploadDate'))
3854             or unified_strdate(search_meta('uploadDate')))
3855         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3856             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3857         info['upload_date'] = upload_date
3858
3859         for to, frm in fallbacks.items():
3860             if not info.get(to):
3861                 info[to] = info.get(frm)
3862
3863         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3864             v = info.get(s_k)
3865             if v:
3866                 info[d_k] = v
3867
3868         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3869         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3870         is_membersonly = None
3871         is_premium = None
3872         if initial_data and is_private is not None:
3873             is_membersonly = False
3874             is_premium = False
3875             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3876             badge_labels = set()
3877             for content in contents:
3878                 if not isinstance(content, dict):
3879                     continue
3880                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3881             for badge_label in badge_labels:
3882                 if badge_label.lower() == 'members only':
3883                     is_membersonly = True
3884                 elif badge_label.lower() == 'premium':
3885                     is_premium = True
3886                 elif badge_label.lower() == 'unlisted':
3887                     is_unlisted = True
3888
3889         info['availability'] = self._availability(
3890             is_private=is_private,
3891             needs_premium=is_premium,
3892             needs_subscription=is_membersonly,
3893             needs_auth=info['age_limit'] >= 18,
3894             is_unlisted=None if is_private is None else is_unlisted)
3895
3896         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3897
3898         self.mark_watched(video_id, player_responses)
3899
3900         return info
3901
3902
3903 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3904
3905     @staticmethod
3906     def passthrough_smuggled_data(func):
3907         def _smuggle(entries, smuggled_data):
3908             for entry in entries:
3909                 # TODO: Convert URL to music.youtube instead.
3910                 # Do we need to passthrough any other smuggled_data?
3911                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3912                 yield entry
3913
3914         @functools.wraps(func)
3915         def wrapper(self, url):
3916             url, smuggled_data = unsmuggle_url(url, {})
3917             if self.is_music_url(url):
3918                 smuggled_data['is_music_url'] = True
3919             info_dict = func(self, url, smuggled_data)
3920             if smuggled_data and info_dict.get('entries'):
3921                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3922             return info_dict
3923         return wrapper
3924
3925     def _extract_channel_id(self, webpage):
3926         channel_id = self._html_search_meta(
3927             'channelId', webpage, 'channel id', default=None)
3928         if channel_id:
3929             return channel_id
3930         channel_url = self._html_search_meta(
3931             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3932              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3933              'twitter:app:url:googleplay'), webpage, 'channel url')
3934         return self._search_regex(
3935             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3936             channel_url, 'channel id')
3937
3938     @staticmethod
3939     def _extract_basic_item_renderer(item):
3940         # Modified from _extract_grid_item_renderer
3941         known_basic_renderers = (
3942             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3943         )
3944         for key, renderer in item.items():
3945             if not isinstance(renderer, dict):
3946                 continue
3947             elif key in known_basic_renderers:
3948                 return renderer
3949             elif key.startswith('grid') and key.endswith('Renderer'):
3950                 return renderer
3951
3952     def _grid_entries(self, grid_renderer):
3953         for item in grid_renderer['items']:
3954             if not isinstance(item, dict):
3955                 continue
3956             renderer = self._extract_basic_item_renderer(item)
3957             if not isinstance(renderer, dict):
3958                 continue
3959             title = self._get_text(renderer, 'title')
3960
3961             # playlist
3962             playlist_id = renderer.get('playlistId')
3963             if playlist_id:
3964                 yield self.url_result(
3965                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3966                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3967                     video_title=title)
3968                 continue
3969             # video
3970             video_id = renderer.get('videoId')
3971             if video_id:
3972                 yield self._extract_video(renderer)
3973                 continue
3974             # channel
3975             channel_id = renderer.get('channelId')
3976             if channel_id:
3977                 yield self.url_result(
3978                     'https://www.youtube.com/channel/%s' % channel_id,
3979                     ie=YoutubeTabIE.ie_key(), video_title=title)
3980                 continue
3981             # generic endpoint URL support
3982             ep_url = urljoin('https://www.youtube.com/', try_get(
3983                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3984                 str))
3985             if ep_url:
3986                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3987                     if ie.suitable(ep_url):
3988                         yield self.url_result(
3989                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3990                         break
3991
3992     def _music_reponsive_list_entry(self, renderer):
3993         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3994         if video_id:
3995             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3996                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3997         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3998         if playlist_id:
3999             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4000             if video_id:
4001                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4002                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4003             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4004                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4005         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4006         if browse_id:
4007             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4008                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4009
4010     def _shelf_entries_from_content(self, shelf_renderer):
4011         content = shelf_renderer.get('content')
4012         if not isinstance(content, dict):
4013             return
4014         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4015         if renderer:
4016             # TODO: add support for nested playlists so each shelf is processed
4017             # as separate playlist
4018             # TODO: this includes only first N items
4019             yield from self._grid_entries(renderer)
4020         renderer = content.get('horizontalListRenderer')
4021         if renderer:
4022             # TODO
4023             pass
4024
4025     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4026         ep = try_get(
4027             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4028             str)
4029         shelf_url = urljoin('https://www.youtube.com', ep)
4030         if shelf_url:
4031             # Skipping links to another channels, note that checking for
4032             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4033             # will not work
4034             if skip_channels and '/channels?' in shelf_url:
4035                 return
4036             title = self._get_text(shelf_renderer, 'title')
4037             yield self.url_result(shelf_url, video_title=title)
4038         # Shelf may not contain shelf URL, fallback to extraction from content
4039         yield from self._shelf_entries_from_content(shelf_renderer)
4040
4041     def _playlist_entries(self, video_list_renderer):
4042         for content in video_list_renderer['contents']:
4043             if not isinstance(content, dict):
4044                 continue
4045             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4046             if not isinstance(renderer, dict):
4047                 continue
4048             video_id = renderer.get('videoId')
4049             if not video_id:
4050                 continue
4051             yield self._extract_video(renderer)
4052
4053     def _rich_entries(self, rich_grid_renderer):
4054         renderer = try_get(
4055             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
4056         video_id = renderer.get('videoId')
4057         if not video_id:
4058             return
4059         yield self._extract_video(renderer)
4060
4061     def _video_entry(self, video_renderer):
4062         video_id = video_renderer.get('videoId')
4063         if video_id:
4064             return self._extract_video(video_renderer)
4065
4066     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4067         url = urljoin('https://youtube.com', traverse_obj(
4068             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4069         if url:
4070             return self.url_result(
4071                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4072
4073     def _post_thread_entries(self, post_thread_renderer):
4074         post_renderer = try_get(
4075             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4076         if not post_renderer:
4077             return
4078         # video attachment
4079         video_renderer = try_get(
4080             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4081         video_id = video_renderer.get('videoId')
4082         if video_id:
4083             entry = self._extract_video(video_renderer)
4084             if entry:
4085                 yield entry
4086         # playlist attachment
4087         playlist_id = try_get(
4088             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4089         if playlist_id:
4090             yield self.url_result(
4091                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4092                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4093         # inline video links
4094         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4095         for run in runs:
4096             if not isinstance(run, dict):
4097                 continue
4098             ep_url = try_get(
4099                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4100             if not ep_url:
4101                 continue
4102             if not YoutubeIE.suitable(ep_url):
4103                 continue
4104             ep_video_id = YoutubeIE._match_id(ep_url)
4105             if video_id == ep_video_id:
4106                 continue
4107             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4108
4109     def _post_thread_continuation_entries(self, post_thread_continuation):
4110         contents = post_thread_continuation.get('contents')
4111         if not isinstance(contents, list):
4112             return
4113         for content in contents:
4114             renderer = content.get('backstagePostThreadRenderer')
4115             if isinstance(renderer, dict):
4116                 yield from self._post_thread_entries(renderer)
4117                 continue
4118             renderer = content.get('videoRenderer')
4119             if isinstance(renderer, dict):
4120                 yield self._video_entry(renderer)
4121
4122     r''' # unused
4123     def _rich_grid_entries(self, contents):
4124         for content in contents:
4125             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4126             if video_renderer:
4127                 entry = self._video_entry(video_renderer)
4128                 if entry:
4129                     yield entry
4130     '''
4131
4132     def _extract_entries(self, parent_renderer, continuation_list):
4133         # continuation_list is modified in-place with continuation_list = [continuation_token]
4134         continuation_list[:] = [None]
4135         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4136         for content in contents:
4137             if not isinstance(content, dict):
4138                 continue
4139             is_renderer = traverse_obj(
4140                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4141                 expected_type=dict)
4142             if not is_renderer:
4143                 renderer = content.get('richItemRenderer')
4144                 if renderer:
4145                     for entry in self._rich_entries(renderer):
4146                         yield entry
4147                     continuation_list[0] = self._extract_continuation(parent_renderer)
4148                 continue
4149             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4150             for isr_content in isr_contents:
4151                 if not isinstance(isr_content, dict):
4152                     continue
4153
4154                 known_renderers = {
4155                     'playlistVideoListRenderer': self._playlist_entries,
4156                     'gridRenderer': self._grid_entries,
4157                     'reelShelfRenderer': self._grid_entries,
4158                     'shelfRenderer': self._shelf_entries,
4159                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4160                     'backstagePostThreadRenderer': self._post_thread_entries,
4161                     'videoRenderer': lambda x: [self._video_entry(x)],
4162                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4163                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4164                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4165                 }
4166                 for key, renderer in isr_content.items():
4167                     if key not in known_renderers:
4168                         continue
4169                     for entry in known_renderers[key](renderer):
4170                         if entry:
4171                             yield entry
4172                     continuation_list[0] = self._extract_continuation(renderer)
4173                     break
4174
4175             if not continuation_list[0]:
4176                 continuation_list[0] = self._extract_continuation(is_renderer)
4177
4178         if not continuation_list[0]:
4179             continuation_list[0] = self._extract_continuation(parent_renderer)
4180
4181     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4182         continuation_list = [None]
4183         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4184         tab_content = try_get(tab, lambda x: x['content'], dict)
4185         if not tab_content:
4186             return
4187         parent_renderer = (
4188             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4189             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4190         yield from extract_entries(parent_renderer)
4191         continuation = continuation_list[0]
4192
4193         for page_num in itertools.count(1):
4194             if not continuation:
4195                 break
4196             headers = self.generate_api_headers(
4197                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4198             response = self._extract_response(
4199                 item_id=f'{item_id} page {page_num}',
4200                 query=continuation, headers=headers, ytcfg=ytcfg,
4201                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4202
4203             if not response:
4204                 break
4205             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4206             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4207             visitor_data = self._extract_visitor_data(response) or visitor_data
4208
4209             known_continuation_renderers = {
4210                 'playlistVideoListContinuation': self._playlist_entries,
4211                 'gridContinuation': self._grid_entries,
4212                 'itemSectionContinuation': self._post_thread_continuation_entries,
4213                 'sectionListContinuation': extract_entries,  # for feeds
4214             }
4215             continuation_contents = try_get(
4216                 response, lambda x: x['continuationContents'], dict) or {}
4217             continuation_renderer = None
4218             for key, value in continuation_contents.items():
4219                 if key not in known_continuation_renderers:
4220                     continue
4221                 continuation_renderer = value
4222                 continuation_list = [None]
4223                 yield from known_continuation_renderers[key](continuation_renderer)
4224                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4225                 break
4226             if continuation_renderer:
4227                 continue
4228
4229             known_renderers = {
4230                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4231                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4232                 'gridVideoRenderer': (self._grid_entries, 'items'),
4233                 'gridChannelRenderer': (self._grid_entries, 'items'),
4234                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4235                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4236                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4237                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4238             }
4239             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4240             continuation_items = try_get(
4241                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4242             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4243             video_items_renderer = None
4244             for key, value in continuation_item.items():
4245                 if key not in known_renderers:
4246                     continue
4247                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4248                 continuation_list = [None]
4249                 yield from known_renderers[key][0](video_items_renderer)
4250                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4251                 break
4252             if video_items_renderer:
4253                 continue
4254             break
4255
4256     @staticmethod
4257     def _extract_selected_tab(tabs, fatal=True):
4258         for tab in tabs:
4259             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4260             if renderer.get('selected') is True:
4261                 return renderer
4262         else:
4263             if fatal:
4264                 raise ExtractorError('Unable to find selected tab')
4265
4266     def _extract_uploader(self, data):
4267         uploader = {}
4268         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4269         owner = try_get(
4270             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4271         if owner:
4272             owner_text = owner.get('text')
4273             uploader['uploader'] = self._search_regex(
4274                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4275             uploader['uploader_id'] = try_get(
4276                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], str)
4277             uploader['uploader_url'] = urljoin(
4278                 'https://www.youtube.com/',
4279                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], str))
4280         return {k: v for k, v in uploader.items() if v is not None}
4281
4282     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4283         playlist_id = title = description = channel_url = channel_name = channel_id = None
4284         tags = []
4285
4286         selected_tab = self._extract_selected_tab(tabs)
4287         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4288         renderer = try_get(
4289             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4290         if renderer:
4291             channel_name = renderer.get('title')
4292             channel_url = renderer.get('channelUrl')
4293             channel_id = renderer.get('externalId')
4294         else:
4295             renderer = try_get(
4296                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4297
4298         if renderer:
4299             title = renderer.get('title')
4300             description = renderer.get('description', '')
4301             playlist_id = channel_id
4302             tags = renderer.get('keywords', '').split()
4303
4304         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4305         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4306         def _get_uncropped(url):
4307             return url_or_none((url or '').split('=')[0] + '=s0')
4308
4309         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4310         if avatar_thumbnails:
4311             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4312             if uncropped_avatar:
4313                 avatar_thumbnails.append({
4314                     'url': uncropped_avatar,
4315                     'id': 'avatar_uncropped',
4316                     'preference': 1
4317                 })
4318
4319         channel_banners = self._extract_thumbnails(
4320             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4321         for banner in channel_banners:
4322             banner['preference'] = -10
4323
4324         if channel_banners:
4325             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4326             if uncropped_banner:
4327                 channel_banners.append({
4328                     'url': uncropped_banner,
4329                     'id': 'banner_uncropped',
4330                     'preference': -5
4331                 })
4332
4333         primary_thumbnails = self._extract_thumbnails(
4334             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4335
4336         if playlist_id is None:
4337             playlist_id = item_id
4338
4339         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4340         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4341         if title is None:
4342             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4343         title += format_field(selected_tab, 'title', ' - %s')
4344         title += format_field(selected_tab, 'expandedText', ' - %s')
4345
4346         metadata = {
4347             'playlist_id': playlist_id,
4348             'playlist_title': title,
4349             'playlist_description': description,
4350             'uploader': channel_name,
4351             'uploader_id': channel_id,
4352             'uploader_url': channel_url,
4353             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4354             'tags': tags,
4355             'view_count': self._get_count(playlist_stats, 1),
4356             'availability': self._extract_availability(data),
4357             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4358             'playlist_count': self._get_count(playlist_stats, 0),
4359             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4360         }
4361         if not channel_id:
4362             metadata.update(self._extract_uploader(data))
4363         metadata.update({
4364             'channel': metadata['uploader'],
4365             'channel_id': metadata['uploader_id'],
4366             'channel_url': metadata['uploader_url']})
4367         return self.playlist_result(
4368             self._entries(
4369                 selected_tab, playlist_id, ytcfg,
4370                 self._extract_account_syncid(ytcfg, data),
4371                 self._extract_visitor_data(data, ytcfg)),
4372             **metadata)
4373
4374     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4375         first_id = last_id = response = None
4376         for page_num in itertools.count(1):
4377             videos = list(self._playlist_entries(playlist))
4378             if not videos:
4379                 return
4380             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4381             if start >= len(videos):
4382                 return
4383             yield from videos[start:]
4384             first_id = first_id or videos[0]['id']
4385             last_id = videos[-1]['id']
4386             watch_endpoint = try_get(
4387                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4388             headers = self.generate_api_headers(
4389                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4390                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4391             query = {
4392                 'playlistId': playlist_id,
4393                 'videoId': watch_endpoint.get('videoId') or last_id,
4394                 'index': watch_endpoint.get('index') or len(videos),
4395                 'params': watch_endpoint.get('params') or 'OAE%3D'
4396             }
4397             response = self._extract_response(
4398                 item_id='%s page %d' % (playlist_id, page_num),
4399                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4400                 check_get_keys='contents'
4401             )
4402             playlist = try_get(
4403                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4404
4405     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4406         title = playlist.get('title') or try_get(
4407             data, lambda x: x['titleText']['simpleText'], str)
4408         playlist_id = playlist.get('playlistId') or item_id
4409
4410         # Delegating everything except mix playlists to regular tab-based playlist URL
4411         playlist_url = urljoin(url, try_get(
4412             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4413             str))
4414
4415         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4416         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4417         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4418
4419         if playlist_url and playlist_url != url and not is_known_unviewable:
4420             return self.url_result(
4421                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4422                 video_title=title)
4423
4424         return self.playlist_result(
4425             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4426             playlist_id=playlist_id, playlist_title=title)
4427
4428     def _extract_availability(self, data):
4429         """
4430         Gets the availability of a given playlist/tab.
4431         Note: Unless YouTube tells us explicitly, we do not assume it is public
4432         @param data: response
4433         """
4434         is_private = is_unlisted = None
4435         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4436         badge_labels = self._extract_badges(renderer)
4437
4438         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4439         privacy_dropdown_entries = try_get(
4440             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4441         for renderer_dict in privacy_dropdown_entries:
4442             is_selected = try_get(
4443                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4444             if not is_selected:
4445                 continue
4446             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4447             if label:
4448                 badge_labels.add(label.lower())
4449                 break
4450
4451         for badge_label in badge_labels:
4452             if badge_label == 'unlisted':
4453                 is_unlisted = True
4454             elif badge_label == 'private':
4455                 is_private = True
4456             elif badge_label == 'public':
4457                 is_unlisted = is_private = False
4458         return self._availability(is_private, False, False, False, is_unlisted)
4459
4460     @staticmethod
4461     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4462         sidebar_renderer = try_get(
4463             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4464         for item in sidebar_renderer:
4465             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4466             if renderer:
4467                 return renderer
4468
4469     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4470         """
4471         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4472         """
4473         browse_id = params = None
4474         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4475         if not renderer:
4476             return
4477         menu_renderer = try_get(
4478             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4479         for menu_item in menu_renderer:
4480             if not isinstance(menu_item, dict):
4481                 continue
4482             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4483             text = try_get(
4484                 nav_item_renderer, lambda x: x['text']['simpleText'], str)
4485             if not text or text.lower() != 'show unavailable videos':
4486                 continue
4487             browse_endpoint = try_get(
4488                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4489             browse_id = browse_endpoint.get('browseId')
4490             params = browse_endpoint.get('params')
4491             break
4492
4493         headers = self.generate_api_headers(
4494             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4495             visitor_data=self._extract_visitor_data(data, ytcfg))
4496         query = {
4497             'params': params or 'wgYCCAA=',
4498             'browseId': browse_id or 'VL%s' % item_id
4499         }
4500         return self._extract_response(
4501             item_id=item_id, headers=headers, query=query,
4502             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4503             note='Downloading API JSON with unavailable videos')
4504
4505     @functools.cached_property
4506     def skip_webpage(self):
4507         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4508
4509     def _extract_webpage(self, url, item_id, fatal=True):
4510         webpage, data = None, None
4511         for retry in self.RetryManager(fatal=fatal):
4512             try:
4513                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
4514                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4515             except ExtractorError as e:
4516                 if isinstance(e.cause, network_exceptions):
4517                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4518                         retry.error = e
4519                         continue
4520                 self._error_or_warning(e, fatal=fatal)
4521                 break
4522
4523             try:
4524                 self._extract_and_report_alerts(data)
4525             except ExtractorError as e:
4526                 self._error_or_warning(e, fatal=fatal)
4527                 break
4528
4529             # Sometimes youtube returns a webpage with incomplete ytInitialData
4530             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4531             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
4532                 retry.error = ExtractorError('Incomplete yt initial data received')
4533                 continue
4534
4535         return webpage, data
4536
4537     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4538         """Use if failed to extract ytcfg (and data) from initial webpage"""
4539         if not ytcfg and self.is_authenticated:
4540             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4541             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4542                 raise ExtractorError(
4543                     f'{msg}. If you are not downloading private content, or '
4544                     'your cookies are only for the first account and channel,'
4545                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4546                     expected=True)
4547             self.report_warning(msg, only_once=True)
4548
4549     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4550         data = None
4551         if not self.skip_webpage:
4552             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4553             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4554             # Reject webpage data if redirected to home page without explicitly requesting
4555             selected_tab = self._extract_selected_tab(traverse_obj(
4556                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4557             if (url != 'https://www.youtube.com/feed/recommended'
4558                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4559                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4560                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4561                 if fatal:
4562                     raise ExtractorError(msg, expected=True)
4563                 self.report_warning(msg, only_once=True)
4564         if not data:
4565             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4566             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4567         return data, ytcfg
4568
4569     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4570         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4571         resolve_response = self._extract_response(
4572             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4573             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4574         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4575         for ep_key, ep in endpoints.items():
4576             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4577             if params:
4578                 return self._extract_response(
4579                     item_id=item_id, query=params, ep=ep, headers=headers,
4580                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4581                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4582         err_note = 'Failed to resolve url (does the playlist exist?)'
4583         if fatal:
4584             raise ExtractorError(err_note, expected=True)
4585         self.report_warning(err_note, item_id)
4586
4587     _SEARCH_PARAMS = None
4588
4589     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4590         data = {'query': query}
4591         if params is NO_DEFAULT:
4592             params = self._SEARCH_PARAMS
4593         if params:
4594             data['params'] = params
4595
4596         content_keys = (
4597             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4598             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4599             # ytmusic search
4600             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4601             ('continuationContents', ),
4602         )
4603         display_id = f'query "{query}"'
4604         check_get_keys = tuple({keys[0] for keys in content_keys})
4605         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4606         self._report_playlist_authcheck(ytcfg, fatal=False)
4607
4608         continuation_list = [None]
4609         search = None
4610         for page_num in itertools.count(1):
4611             data.update(continuation_list[0] or {})
4612             headers = self.generate_api_headers(
4613                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4614             search = self._extract_response(
4615                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4616                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4617             slr_contents = traverse_obj(search, *content_keys)
4618             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4619             if not continuation_list[0]:
4620                 break
4621
4622
4623 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4624     IE_DESC = 'YouTube Tabs'
4625     _VALID_URL = r'''(?x:
4626         https?://
4627             (?:\w+\.)?
4628             (?:
4629                 youtube(?:kids)?\.com|
4630                 %(invidious)s
4631             )/
4632             (?:
4633                 (?P<channel_type>channel|c|user|browse)/|
4634                 (?P<not_channel>
4635                     feed/|hashtag/|
4636                     (?:playlist|watch)\?.*?\blist=
4637                 )|
4638                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4639             )
4640             (?P<id>[^/?\#&]+)
4641     )''' % {
4642         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4643         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4644     }
4645     IE_NAME = 'youtube:tab'
4646
4647     _TESTS = [{
4648         'note': 'playlists, multipage',
4649         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4650         'playlist_mincount': 94,
4651         'info_dict': {
4652             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4653             'title': 'Igor Kleiner - Playlists',
4654             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4655             'uploader': 'Igor Kleiner',
4656             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4657             'channel': 'Igor Kleiner',
4658             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4659             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4660             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4661             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4662             'channel_follower_count': int
4663         },
4664     }, {
4665         'note': 'playlists, multipage, different order',
4666         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4667         'playlist_mincount': 94,
4668         'info_dict': {
4669             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4670             'title': 'Igor Kleiner - Playlists',
4671             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4672             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4673             'uploader': 'Igor Kleiner',
4674             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4675             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4676             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4677             'channel': 'Igor Kleiner',
4678             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4679             'channel_follower_count': int
4680         },
4681     }, {
4682         'note': 'playlists, series',
4683         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4684         'playlist_mincount': 5,
4685         'info_dict': {
4686             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4687             'title': '3Blue1Brown - Playlists',
4688             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4689             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4690             'uploader': '3Blue1Brown',
4691             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4692             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4693             'channel': '3Blue1Brown',
4694             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4695             'tags': ['Mathematics'],
4696             'channel_follower_count': int
4697         },
4698     }, {
4699         'note': 'playlists, singlepage',
4700         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4701         'playlist_mincount': 4,
4702         'info_dict': {
4703             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4704             'title': 'ThirstForScience - Playlists',
4705             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4706             'uploader': 'ThirstForScience',
4707             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4708             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4709             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4710             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4711             'tags': 'count:13',
4712             'channel': 'ThirstForScience',
4713             'channel_follower_count': int
4714         }
4715     }, {
4716         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4717         'only_matching': True,
4718     }, {
4719         'note': 'basic, single video playlist',
4720         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4721         'info_dict': {
4722             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4723             'uploader': 'Sergey M.',
4724             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4725             'title': 'youtube-dl public playlist',
4726             'description': '',
4727             'tags': [],
4728             'view_count': int,
4729             'modified_date': '20201130',
4730             'channel': 'Sergey M.',
4731             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4732             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4733             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4734         },
4735         'playlist_count': 1,
4736     }, {
4737         'note': 'empty playlist',
4738         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4739         'info_dict': {
4740             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4741             'uploader': 'Sergey M.',
4742             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4743             'title': 'youtube-dl empty playlist',
4744             'tags': [],
4745             'channel': 'Sergey M.',
4746             'description': '',
4747             'modified_date': '20160902',
4748             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4749             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4750             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4751         },
4752         'playlist_count': 0,
4753     }, {
4754         'note': 'Home tab',
4755         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4756         'info_dict': {
4757             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4758             'title': 'lex will - Home',
4759             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4760             'uploader': 'lex will',
4761             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4762             'channel': 'lex will',
4763             'tags': ['bible', 'history', 'prophesy'],
4764             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4765             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4766             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4767             'channel_follower_count': int
4768         },
4769         'playlist_mincount': 2,
4770     }, {
4771         'note': 'Videos tab',
4772         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4773         'info_dict': {
4774             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4775             'title': 'lex will - Videos',
4776             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4777             'uploader': 'lex will',
4778             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4779             'tags': ['bible', 'history', 'prophesy'],
4780             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4781             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4782             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4783             'channel': 'lex will',
4784             'channel_follower_count': int
4785         },
4786         'playlist_mincount': 975,
4787     }, {
4788         'note': 'Videos tab, sorted by popular',
4789         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4790         'info_dict': {
4791             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4792             'title': 'lex will - Videos',
4793             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4794             'uploader': 'lex will',
4795             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4796             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4797             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4798             'channel': 'lex will',
4799             'tags': ['bible', 'history', 'prophesy'],
4800             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4801             'channel_follower_count': int
4802         },
4803         'playlist_mincount': 199,
4804     }, {
4805         'note': 'Playlists tab',
4806         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4807         'info_dict': {
4808             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4809             'title': 'lex will - Playlists',
4810             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4811             'uploader': 'lex will',
4812             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4813             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4814             'channel': 'lex will',
4815             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4816             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4817             'tags': ['bible', 'history', 'prophesy'],
4818             'channel_follower_count': int
4819         },
4820         'playlist_mincount': 17,
4821     }, {
4822         'note': 'Community tab',
4823         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4824         'info_dict': {
4825             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4826             'title': 'lex will - Community',
4827             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4828             'uploader': 'lex will',
4829             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4830             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4831             'channel': 'lex will',
4832             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4833             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4834             'tags': ['bible', 'history', 'prophesy'],
4835             'channel_follower_count': int
4836         },
4837         'playlist_mincount': 18,
4838     }, {
4839         'note': 'Channels tab',
4840         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4841         'info_dict': {
4842             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4843             'title': 'lex will - Channels',
4844             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4845             'uploader': 'lex will',
4846             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4847             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4848             'channel': 'lex will',
4849             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4850             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4851             'tags': ['bible', 'history', 'prophesy'],
4852             'channel_follower_count': int
4853         },
4854         'playlist_mincount': 12,
4855     }, {
4856         'note': 'Search tab',
4857         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4858         'playlist_mincount': 40,
4859         'info_dict': {
4860             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4861             'title': '3Blue1Brown - Search - linear algebra',
4862             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4863             'uploader': '3Blue1Brown',
4864             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4865             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4866             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4867             'tags': ['Mathematics'],
4868             'channel': '3Blue1Brown',
4869             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4870             'channel_follower_count': int
4871         },
4872     }, {
4873         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4874         'only_matching': True,
4875     }, {
4876         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4877         'only_matching': True,
4878     }, {
4879         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4880         'only_matching': True,
4881     }, {
4882         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4883         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4884         'info_dict': {
4885             'title': '29C3: Not my department',
4886             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4887             'uploader': 'Christiaan008',
4888             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4889             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4890             'tags': [],
4891             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4892             'view_count': int,
4893             'modified_date': '20150605',
4894             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4895             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4896             'channel': 'Christiaan008',
4897         },
4898         'playlist_count': 96,
4899     }, {
4900         'note': 'Large playlist',
4901         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4902         'info_dict': {
4903             'title': 'Uploads from Cauchemar',
4904             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4905             'uploader': 'Cauchemar',
4906             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4907             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4908             'tags': [],
4909             'modified_date': r're:\d{8}',
4910             'channel': 'Cauchemar',
4911             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4912             'view_count': int,
4913             'description': '',
4914             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4915         },
4916         'playlist_mincount': 1123,
4917         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4918     }, {
4919         'note': 'even larger playlist, 8832 videos',
4920         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4921         'only_matching': True,
4922     }, {
4923         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4924         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4925         'info_dict': {
4926             'title': 'Uploads from Interstellar Movie',
4927             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4928             'uploader': 'Interstellar Movie',
4929             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4930             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4931             'tags': [],
4932             'view_count': int,
4933             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4934             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4935             'channel': 'Interstellar Movie',
4936             'description': '',
4937             'modified_date': r're:\d{8}',
4938         },
4939         'playlist_mincount': 21,
4940     }, {
4941         'note': 'Playlist with "show unavailable videos" button',
4942         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4943         'info_dict': {
4944             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4945             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4946             'uploader': 'Phim Siêu Nhân Nhật Bản',
4947             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4948             'view_count': int,
4949             'channel': 'Phim Siêu Nhân Nhật Bản',
4950             'tags': [],
4951             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4952             'description': '',
4953             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4954             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4955             'modified_date': r're:\d{8}',
4956         },
4957         'playlist_mincount': 200,
4958         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4959     }, {
4960         'note': 'Playlist with unavailable videos in page 7',
4961         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4962         'info_dict': {
4963             'title': 'Uploads from BlankTV',
4964             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4965             'uploader': 'BlankTV',
4966             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4967             'channel': 'BlankTV',
4968             'channel_url': 'https://www.youtube.com/c/blanktv',
4969             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4970             'view_count': int,
4971             'tags': [],
4972             'uploader_url': 'https://www.youtube.com/c/blanktv',
4973             'modified_date': r're:\d{8}',
4974             'description': '',
4975         },
4976         'playlist_mincount': 1000,
4977         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4978     }, {
4979         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4980         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4981         'info_dict': {
4982             'title': 'Data Analysis with Dr Mike Pound',
4983             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4984             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4985             'uploader': 'Computerphile',
4986             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4987             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4988             'tags': [],
4989             'view_count': int,
4990             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4991             'channel_url': 'https://www.youtube.com/user/Computerphile',
4992             'channel': 'Computerphile',
4993         },
4994         'playlist_mincount': 11,
4995     }, {
4996         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4997         'only_matching': True,
4998     }, {
4999         'note': 'Playlist URL that does not actually serve a playlist',
5000         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5001         'info_dict': {
5002             'id': 'FqZTN594JQw',
5003             'ext': 'webm',
5004             'title': "Smiley's People 01 detective, Adventure Series, Action",
5005             'uploader': 'STREEM',
5006             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5007             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5008             'upload_date': '20150526',
5009             'license': 'Standard YouTube License',
5010             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5011             'categories': ['People & Blogs'],
5012             'tags': list,
5013             'view_count': int,
5014             'like_count': int,
5015         },
5016         'params': {
5017             'skip_download': True,
5018         },
5019         'skip': 'This video is not available.',
5020         'add_ie': [YoutubeIE.ie_key()],
5021     }, {
5022         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5023         'only_matching': True,
5024     }, {
5025         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5026         'only_matching': True,
5027     }, {
5028         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5029         'info_dict': {
5030             'id': 'Wq15eF5vCbI',  # This will keep changing
5031             'ext': 'mp4',
5032             'title': str,
5033             'uploader': 'Sky News',
5034             'uploader_id': 'skynews',
5035             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5036             'upload_date': r're:\d{8}',
5037             'description': str,
5038             'categories': ['News & Politics'],
5039             'tags': list,
5040             'like_count': int,
5041             'release_timestamp': 1642502819,
5042             'channel': 'Sky News',
5043             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5044             'age_limit': 0,
5045             'view_count': int,
5046             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5047             'playable_in_embed': True,
5048             'release_date': '20220118',
5049             'availability': 'public',
5050             'live_status': 'is_live',
5051             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5052             'channel_follower_count': int
5053         },
5054         'params': {
5055             'skip_download': True,
5056         },
5057         'expected_warnings': ['Ignoring subtitle tracks found in '],
5058     }, {
5059         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5060         'info_dict': {
5061             'id': 'a48o2S1cPoo',
5062             'ext': 'mp4',
5063             'title': 'The Young Turks - Live Main Show',
5064             'uploader': 'The Young Turks',
5065             'uploader_id': 'TheYoungTurks',
5066             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5067             'upload_date': '20150715',
5068             'license': 'Standard YouTube License',
5069             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5070             'categories': ['News & Politics'],
5071             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5072             'like_count': int,
5073         },
5074         'params': {
5075             'skip_download': True,
5076         },
5077         'only_matching': True,
5078     }, {
5079         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5080         'only_matching': True,
5081     }, {
5082         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5083         'only_matching': True,
5084     }, {
5085         'note': 'A channel that is not live. Should raise error',
5086         'url': 'https://www.youtube.com/user/numberphile/live',
5087         'only_matching': True,
5088     }, {
5089         'url': 'https://www.youtube.com/feed/trending',
5090         'only_matching': True,
5091     }, {
5092         'url': 'https://www.youtube.com/feed/library',
5093         'only_matching': True,
5094     }, {
5095         'url': 'https://www.youtube.com/feed/history',
5096         'only_matching': True,
5097     }, {
5098         'url': 'https://www.youtube.com/feed/subscriptions',
5099         'only_matching': True,
5100     }, {
5101         'url': 'https://www.youtube.com/feed/watch_later',
5102         'only_matching': True,
5103     }, {
5104         'note': 'Recommended - redirects to home page.',
5105         'url': 'https://www.youtube.com/feed/recommended',
5106         'only_matching': True,
5107     }, {
5108         'note': 'inline playlist with not always working continuations',
5109         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5110         'only_matching': True,
5111     }, {
5112         'url': 'https://www.youtube.com/course',
5113         'only_matching': True,
5114     }, {
5115         'url': 'https://www.youtube.com/zsecurity',
5116         'only_matching': True,
5117     }, {
5118         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5119         'only_matching': True,
5120     }, {
5121         'url': 'https://www.youtube.com/TheYoungTurks/live',
5122         'only_matching': True,
5123     }, {
5124         'url': 'https://www.youtube.com/hashtag/cctv9',
5125         'info_dict': {
5126             'id': 'cctv9',
5127             'title': '#cctv9',
5128             'tags': [],
5129         },
5130         'playlist_mincount': 350,
5131     }, {
5132         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5133         'only_matching': True,
5134     }, {
5135         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5136         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5137         'only_matching': True
5138     }, {
5139         'note': '/browse/ should redirect to /channel/',
5140         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5141         'only_matching': True
5142     }, {
5143         'note': 'VLPL, should redirect to playlist?list=PL...',
5144         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5145         'info_dict': {
5146             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5147             'uploader': 'NoCopyrightSounds',
5148             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5149             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5150             'title': 'NCS : All Releases 💿',
5151             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5152             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5153             'modified_date': r're:\d{8}',
5154             'view_count': int,
5155             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5156             'tags': [],
5157             'channel': 'NoCopyrightSounds',
5158         },
5159         'playlist_mincount': 166,
5160         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5161     }, {
5162         'note': 'Topic, should redirect to playlist?list=UU...',
5163         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5164         'info_dict': {
5165             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5166             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5167             'title': 'Uploads from Royalty Free Music - Topic',
5168             'uploader': 'Royalty Free Music - Topic',
5169             'tags': [],
5170             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5171             'channel': 'Royalty Free Music - Topic',
5172             'view_count': int,
5173             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5174             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5175             'modified_date': r're:\d{8}',
5176             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5177             'description': '',
5178         },
5179         'expected_warnings': [
5180             'The URL does not have a videos tab',
5181             r'[Uu]navailable videos (are|will be) hidden',
5182         ],
5183         'playlist_mincount': 101,
5184     }, {
5185         'note': 'Topic without a UU playlist',
5186         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5187         'info_dict': {
5188             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5189             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5190             'tags': [],
5191         },
5192         'expected_warnings': [
5193             'the playlist redirect gave error',
5194         ],
5195         'playlist_mincount': 9,
5196     }, {
5197         'note': 'Youtube music Album',
5198         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5199         'info_dict': {
5200             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5201             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5202             'tags': [],
5203             'view_count': int,
5204             'description': '',
5205             'availability': 'unlisted',
5206             'modified_date': r're:\d{8}',
5207         },
5208         'playlist_count': 50,
5209     }, {
5210         'note': 'unlisted single video playlist',
5211         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5212         'info_dict': {
5213             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5214             'uploader': 'colethedj',
5215             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5216             'title': 'yt-dlp unlisted playlist test',
5217             'availability': 'unlisted',
5218             'tags': [],
5219             'modified_date': '20220418',
5220             'channel': 'colethedj',
5221             'view_count': int,
5222             'description': '',
5223             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5224             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5225             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5226         },
5227         'playlist_count': 1,
5228     }, {
5229         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5230         'url': 'https://www.youtube.com/feed/recommended',
5231         'info_dict': {
5232             'id': 'recommended',
5233             'title': 'recommended',
5234             'tags': [],
5235         },
5236         'playlist_mincount': 50,
5237         'params': {
5238             'skip_download': True,
5239             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5240         },
5241     }, {
5242         'note': 'API Fallback: /videos tab, sorted by oldest first',
5243         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5244         'info_dict': {
5245             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5246             'title': 'Cody\'sLab - Videos',
5247             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5248             'uploader': 'Cody\'sLab',
5249             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5250             'channel': 'Cody\'sLab',
5251             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5252             'tags': [],
5253             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5254             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5255             'channel_follower_count': int
5256         },
5257         'playlist_mincount': 650,
5258         'params': {
5259             'skip_download': True,
5260             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5261         },
5262     }, {
5263         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5264         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5265         'info_dict': {
5266             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5267             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5268             'title': 'Uploads from Royalty Free Music - Topic',
5269             'uploader': 'Royalty Free Music - Topic',
5270             'modified_date': r're:\d{8}',
5271             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5272             'description': '',
5273             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5274             'tags': [],
5275             'channel': 'Royalty Free Music - Topic',
5276             'view_count': int,
5277             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5278         },
5279         'expected_warnings': [
5280             'does not have a videos tab',
5281             r'[Uu]navailable videos (are|will be) hidden',
5282         ],
5283         'playlist_mincount': 101,
5284         'params': {
5285             'skip_download': True,
5286             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5287         },
5288     }, {
5289         'note': 'non-standard redirect to regional channel',
5290         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5291         'only_matching': True
5292     }, {
5293         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5294         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5295         'info_dict': {
5296             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5297             'modified_date': '20220407',
5298             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5299             'tags': [],
5300             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5301             'uploader': 'pukkandan',
5302             'availability': 'unlisted',
5303             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5304             'channel': 'pukkandan',
5305             'description': 'Test for collaborative playlist',
5306             'title': 'yt-dlp test - collaborative playlist',
5307             'view_count': int,
5308             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5309         },
5310         'playlist_mincount': 2
5311     }]
5312
5313     @classmethod
5314     def suitable(cls, url):
5315         return False if YoutubeIE.suitable(url) else super().suitable(url)
5316
5317     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5318
5319     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5320     def _real_extract(self, url, smuggled_data):
5321         item_id = self._match_id(url)
5322         url = urllib.parse.urlunparse(
5323             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5324         compat_opts = self.get_param('compat_opts', [])
5325
5326         def get_mobj(url):
5327             mobj = self._URL_RE.match(url).groupdict()
5328             mobj.update((k, '') for k, v in mobj.items() if v is None)
5329             return mobj
5330
5331         mobj, redirect_warning = get_mobj(url), None
5332         # Youtube returns incomplete data if tabname is not lower case
5333         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5334         if is_channel:
5335             if smuggled_data.get('is_music_url'):
5336                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5337                     item_id = item_id[2:]
5338                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5339                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5340                     mdata = self._extract_tab_endpoint(
5341                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5342                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5343                                         get_all=False, expected_type=str)
5344                     if not murl:
5345                         raise ExtractorError('Failed to resolve album to playlist')
5346                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5347                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5348                     pre = f'https://www.youtube.com/channel/{item_id}'
5349
5350         original_tab_name = tab
5351         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5352             # Home URLs should redirect to /videos/
5353             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5354                                 'To download only the videos in the home page, add a "/featured" to the URL')
5355             tab = '/videos'
5356
5357         url = ''.join((pre, tab, post))
5358         mobj = get_mobj(url)
5359
5360         # Handle both video/playlist URLs
5361         qs = parse_qs(url)
5362         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5363
5364         if not video_id and mobj['not_channel'].startswith('watch'):
5365             if not playlist_id:
5366                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5367                 raise ExtractorError('Unable to recognize tab page')
5368             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5369             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5370             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5371             mobj = get_mobj(url)
5372
5373         if video_id and playlist_id:
5374             if self.get_param('noplaylist'):
5375                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5376                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5377                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5378             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5379
5380         data, ytcfg = self._extract_data(url, item_id)
5381
5382         # YouTube may provide a non-standard redirect to the regional channel
5383         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5384         redirect_url = traverse_obj(
5385             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5386         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5387             redirect_url = ''.join((
5388                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5389             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5390             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5391
5392         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5393         if tabs:
5394             selected_tab = self._extract_selected_tab(tabs)
5395             selected_tab_name = selected_tab.get('title', '').lower()
5396             if selected_tab_name == 'home':
5397                 selected_tab_name = 'featured'
5398             requested_tab_name = mobj['tab'][1:]
5399             if 'no-youtube-channel-redirect' not in compat_opts:
5400                 if requested_tab_name == 'live':  # Live tab should have redirected to the video
5401                     raise UserNotLive(video_id=mobj['id'])
5402                 if requested_tab_name not in ('', selected_tab_name):
5403                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5404                     if not original_tab_name:
5405                         if item_id[:2] == 'UC':
5406                             # Topic channels don't have /videos. Use the equivalent playlist instead
5407                             pl_id = f'UU{item_id[2:]}'
5408                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5409                             try:
5410                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5411                             except ExtractorError:
5412                                 redirect_warning += ' and the playlist redirect gave error'
5413                             else:
5414                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5415                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5416                         if selected_tab_name and selected_tab_name != requested_tab_name:
5417                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5418                     else:
5419                         raise ExtractorError(redirect_warning, expected=True)
5420
5421         if redirect_warning:
5422             self.to_screen(redirect_warning)
5423         self.write_debug(f'Final URL: {url}')
5424
5425         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5426         if 'no-youtube-unavailable-videos' not in compat_opts:
5427             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5428         self._extract_and_report_alerts(data, only_once=True)
5429         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5430         if tabs:
5431             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5432
5433         playlist = traverse_obj(
5434             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5435         if playlist:
5436             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5437
5438         video_id = traverse_obj(
5439             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5440         if video_id:
5441             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5442                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5443             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5444                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5445
5446         raise ExtractorError('Unable to recognize tab page')
5447
5448
5449 class YoutubePlaylistIE(InfoExtractor):
5450     IE_DESC = 'YouTube playlists'
5451     _VALID_URL = r'''(?x)(?:
5452                         (?:https?://)?
5453                         (?:\w+\.)?
5454                         (?:
5455                             (?:
5456                                 youtube(?:kids)?\.com|
5457                                 %(invidious)s
5458                             )
5459                             /.*?\?.*?\blist=
5460                         )?
5461                         (?P<id>%(playlist_id)s)
5462                      )''' % {
5463         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5464         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5465     }
5466     IE_NAME = 'youtube:playlist'
5467     _TESTS = [{
5468         'note': 'issue #673',
5469         'url': 'PLBB231211A4F62143',
5470         'info_dict': {
5471             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5472             'id': 'PLBB231211A4F62143',
5473             'uploader': 'Wickman',
5474             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5475             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5476             'view_count': int,
5477             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5478             'modified_date': r're:\d{8}',
5479             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5480             'channel': 'Wickman',
5481             'tags': [],
5482             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5483         },
5484         'playlist_mincount': 29,
5485     }, {
5486         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5487         'info_dict': {
5488             'title': 'YDL_safe_search',
5489             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5490         },
5491         'playlist_count': 2,
5492         'skip': 'This playlist is private',
5493     }, {
5494         'note': 'embedded',
5495         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5496         'playlist_count': 4,
5497         'info_dict': {
5498             'title': 'JODA15',
5499             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5500             'uploader': 'milan',
5501             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5502             'description': '',
5503             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5504             'tags': [],
5505             'modified_date': '20140919',
5506             'view_count': int,
5507             'channel': 'milan',
5508             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5509             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5510         },
5511         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5512     }, {
5513         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5514         'playlist_mincount': 455,
5515         'info_dict': {
5516             'title': '2018 Chinese New Singles (11/6 updated)',
5517             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5518             'uploader': 'LBK',
5519             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5520             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5521             'channel': 'LBK',
5522             'view_count': int,
5523             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5524             'tags': [],
5525             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5526             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5527             'modified_date': r're:\d{8}',
5528         },
5529         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5530     }, {
5531         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5532         'only_matching': True,
5533     }, {
5534         # music album playlist
5535         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5536         'only_matching': True,
5537     }]
5538
5539     @classmethod
5540     def suitable(cls, url):
5541         if YoutubeTabIE.suitable(url):
5542             return False
5543         from ..utils import parse_qs
5544         qs = parse_qs(url)
5545         if qs.get('v', [None])[0]:
5546             return False
5547         return super().suitable(url)
5548
5549     def _real_extract(self, url):
5550         playlist_id = self._match_id(url)
5551         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5552         url = update_url_query(
5553             'https://www.youtube.com/playlist',
5554             parse_qs(url) or {'list': playlist_id})
5555         if is_music_url:
5556             url = smuggle_url(url, {'is_music_url': True})
5557         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5558
5559
5560 class YoutubeYtBeIE(InfoExtractor):
5561     IE_DESC = 'youtu.be'
5562     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5563     _TESTS = [{
5564         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5565         'info_dict': {
5566             'id': 'yeWKywCrFtk',
5567             'ext': 'mp4',
5568             'title': 'Small Scale Baler and Braiding Rugs',
5569             'uploader': 'Backus-Page House Museum',
5570             'uploader_id': 'backuspagemuseum',
5571             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5572             'upload_date': '20161008',
5573             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5574             'categories': ['Nonprofits & Activism'],
5575             'tags': list,
5576             'like_count': int,
5577             'age_limit': 0,
5578             'playable_in_embed': True,
5579             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5580             'channel': 'Backus-Page House Museum',
5581             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5582             'live_status': 'not_live',
5583             'view_count': int,
5584             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5585             'availability': 'public',
5586             'duration': 59,
5587             'comment_count': int,
5588             'channel_follower_count': int
5589         },
5590         'params': {
5591             'noplaylist': True,
5592             'skip_download': True,
5593         },
5594     }, {
5595         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5596         'only_matching': True,
5597     }]
5598
5599     def _real_extract(self, url):
5600         mobj = self._match_valid_url(url)
5601         video_id = mobj.group('id')
5602         playlist_id = mobj.group('playlist_id')
5603         return self.url_result(
5604             update_url_query('https://www.youtube.com/watch', {
5605                 'v': video_id,
5606                 'list': playlist_id,
5607                 'feature': 'youtu.be',
5608             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5609
5610
5611 class YoutubeLivestreamEmbedIE(InfoExtractor):
5612     IE_DESC = 'YouTube livestream embeds'
5613     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5614     _TESTS = [{
5615         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5616         'only_matching': True,
5617     }]
5618
5619     def _real_extract(self, url):
5620         channel_id = self._match_id(url)
5621         return self.url_result(
5622             f'https://www.youtube.com/channel/{channel_id}/live',
5623             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5624
5625
5626 class YoutubeYtUserIE(InfoExtractor):
5627     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5628     IE_NAME = 'youtube:user'
5629     _VALID_URL = r'ytuser:(?P<id>.+)'
5630     _TESTS = [{
5631         'url': 'ytuser:phihag',
5632         'only_matching': True,
5633     }]
5634
5635     def _real_extract(self, url):
5636         user_id = self._match_id(url)
5637         return self.url_result(
5638             'https://www.youtube.com/user/%s/videos' % user_id,
5639             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5640
5641
5642 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5643     IE_NAME = 'youtube:favorites'
5644     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5645     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5646     _LOGIN_REQUIRED = True
5647     _TESTS = [{
5648         'url': ':ytfav',
5649         'only_matching': True,
5650     }, {
5651         'url': ':ytfavorites',
5652         'only_matching': True,
5653     }]
5654
5655     def _real_extract(self, url):
5656         return self.url_result(
5657             'https://www.youtube.com/playlist?list=LL',
5658             ie=YoutubeTabIE.ie_key())
5659
5660
5661 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5662     IE_NAME = 'youtube:notif'
5663     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5664     _VALID_URL = r':ytnotif(?:ication)?s?'
5665     _LOGIN_REQUIRED = True
5666     _TESTS = [{
5667         'url': ':ytnotif',
5668         'only_matching': True,
5669     }, {
5670         'url': ':ytnotifications',
5671         'only_matching': True,
5672     }]
5673
5674     def _extract_notification_menu(self, response, continuation_list):
5675         notification_list = traverse_obj(
5676             response,
5677             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5678             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5679             expected_type=list) or []
5680         continuation_list[0] = None
5681         for item in notification_list:
5682             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5683             if entry:
5684                 yield entry
5685             continuation = item.get('continuationItemRenderer')
5686             if continuation:
5687                 continuation_list[0] = continuation
5688
5689     def _extract_notification_renderer(self, notification):
5690         video_id = traverse_obj(
5691             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5692         url = f'https://www.youtube.com/watch?v={video_id}'
5693         channel_id = None
5694         if not video_id:
5695             browse_ep = traverse_obj(
5696                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5697             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5698             post_id = self._search_regex(
5699                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5700                 'post id', default=None)
5701             if not channel_id or not post_id:
5702                 return
5703             # The direct /post url redirects to this in the browser
5704             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5705
5706         channel = traverse_obj(
5707             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5708             expected_type=str)
5709         notification_title = self._get_text(notification, 'shortMessage')
5710         if notification_title:
5711             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
5712         # TODO: handle recommended videos
5713         title = self._search_regex(
5714             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
5715             'video title', default=None)
5716         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5717                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5718                        else None)
5719         return {
5720             '_type': 'url',
5721             'url': url,
5722             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5723             'video_id': video_id,
5724             'title': title,
5725             'channel_id': channel_id,
5726             'channel': channel,
5727             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5728             'upload_date': upload_date,
5729         }
5730
5731     def _notification_menu_entries(self, ytcfg):
5732         continuation_list = [None]
5733         response = None
5734         for page in itertools.count(1):
5735             ctoken = traverse_obj(
5736                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5737             response = self._extract_response(
5738                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5739                 ep='notification/get_notification_menu', check_get_keys='actions',
5740                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5741             yield from self._extract_notification_menu(response, continuation_list)
5742             if not continuation_list[0]:
5743                 break
5744
5745     def _real_extract(self, url):
5746         display_id = 'notifications'
5747         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5748         self._report_playlist_authcheck(ytcfg)
5749         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5750
5751
5752 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5753     IE_DESC = 'YouTube search'
5754     IE_NAME = 'youtube:search'
5755     _SEARCH_KEY = 'ytsearch'
5756     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5757     _TESTS = [{
5758         'url': 'ytsearch5:youtube-dl test video',
5759         'playlist_count': 5,
5760         'info_dict': {
5761             'id': 'youtube-dl test video',
5762             'title': 'youtube-dl test video',
5763         }
5764     }]
5765
5766
5767 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5768     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5769     _SEARCH_KEY = 'ytsearchdate'
5770     IE_DESC = 'YouTube search, newest videos first'
5771     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5772     _TESTS = [{
5773         'url': 'ytsearchdate5:youtube-dl test video',
5774         'playlist_count': 5,
5775         'info_dict': {
5776             'id': 'youtube-dl test video',
5777             'title': 'youtube-dl test video',
5778         }
5779     }]
5780
5781
5782 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5783     IE_DESC = 'YouTube search URLs with sorting and filter support'
5784     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5785     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5786     _TESTS = [{
5787         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5788         'playlist_mincount': 5,
5789         'info_dict': {
5790             'id': 'youtube-dl test video',
5791             'title': 'youtube-dl test video',
5792         }
5793     }, {
5794         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5795         'playlist_mincount': 5,
5796         'info_dict': {
5797             'id': 'python',
5798             'title': 'python',
5799         }
5800     }, {
5801         'url': 'https://www.youtube.com/results?search_query=%23cats',
5802         'playlist_mincount': 1,
5803         'info_dict': {
5804             'id': '#cats',
5805             'title': '#cats',
5806             # The test suite does not have support for nested playlists
5807             # 'entries': [{
5808             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5809             #     'title': '#cats',
5810             # }],
5811         },
5812     }, {
5813         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5814         'only_matching': True,
5815     }]
5816
5817     def _real_extract(self, url):
5818         qs = parse_qs(url)
5819         query = (qs.get('search_query') or qs.get('q'))[0]
5820         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5821
5822
5823 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5824     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5825     IE_NAME = 'youtube:music:search_url'
5826     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5827     _TESTS = [{
5828         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5829         'playlist_count': 16,
5830         'info_dict': {
5831             'id': 'royalty free music',
5832             'title': 'royalty free music',
5833         }
5834     }, {
5835         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5836         'playlist_mincount': 30,
5837         'info_dict': {
5838             'id': 'royalty free music - songs',
5839             'title': 'royalty free music - songs',
5840         },
5841         'params': {'extract_flat': 'in_playlist'}
5842     }, {
5843         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5844         'playlist_mincount': 30,
5845         'info_dict': {
5846             'id': 'royalty free music - community playlists',
5847             'title': 'royalty free music - community playlists',
5848         },
5849         'params': {'extract_flat': 'in_playlist'}
5850     }]
5851
5852     _SECTIONS = {
5853         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5854         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5855         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5856         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5857         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5858         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5859     }
5860
5861     def _real_extract(self, url):
5862         qs = parse_qs(url)
5863         query = (qs.get('search_query') or qs.get('q'))[0]
5864         params = qs.get('sp', (None,))[0]
5865         if params:
5866             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5867         else:
5868             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
5869             params = self._SECTIONS.get(section)
5870             if not params:
5871                 section = None
5872         title = join_nonempty(query, section, delim=' - ')
5873         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5874
5875
5876 class YoutubeFeedsInfoExtractor(InfoExtractor):
5877     """
5878     Base class for feed extractors
5879     Subclasses must re-define the _FEED_NAME property.
5880     """
5881     _LOGIN_REQUIRED = True
5882     _FEED_NAME = 'feeds'
5883
5884     def _real_initialize(self):
5885         YoutubeBaseInfoExtractor._check_login_required(self)
5886
5887     @classproperty
5888     def IE_NAME(self):
5889         return f'youtube:{self._FEED_NAME}'
5890
5891     def _real_extract(self, url):
5892         return self.url_result(
5893             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5894
5895
5896 class YoutubeWatchLaterIE(InfoExtractor):
5897     IE_NAME = 'youtube:watchlater'
5898     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5899     _VALID_URL = r':ytwatchlater'
5900     _TESTS = [{
5901         'url': ':ytwatchlater',
5902         'only_matching': True,
5903     }]
5904
5905     def _real_extract(self, url):
5906         return self.url_result(
5907             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5908
5909
5910 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5911     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5912     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5913     _FEED_NAME = 'recommended'
5914     _LOGIN_REQUIRED = False
5915     _TESTS = [{
5916         'url': ':ytrec',
5917         'only_matching': True,
5918     }, {
5919         'url': ':ytrecommended',
5920         'only_matching': True,
5921     }, {
5922         'url': 'https://youtube.com',
5923         'only_matching': True,
5924     }]
5925
5926
5927 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5928     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5929     _VALID_URL = r':ytsub(?:scription)?s?'
5930     _FEED_NAME = 'subscriptions'
5931     _TESTS = [{
5932         'url': ':ytsubs',
5933         'only_matching': True,
5934     }, {
5935         'url': ':ytsubscriptions',
5936         'only_matching': True,
5937     }]
5938
5939
5940 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5941     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5942     _VALID_URL = r':ythis(?:tory)?'
5943     _FEED_NAME = 'history'
5944     _TESTS = [{
5945         'url': ':ythistory',
5946         'only_matching': True,
5947     }]
5948
5949
5950 class YoutubeStoriesIE(InfoExtractor):
5951     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5952     IE_NAME = 'youtube:stories'
5953     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5954     _TESTS = [{
5955         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5956         'only_matching': True,
5957     }]
5958
5959     def _real_extract(self, url):
5960         playlist_id = f'RLTD{self._match_id(url)}'
5961         return self.url_result(
5962             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5963             ie=YoutubeTabIE, video_id=playlist_id)
5964
5965
5966 class YoutubeTruncatedURLIE(InfoExtractor):
5967     IE_NAME = 'youtube:truncated_url'
5968     IE_DESC = False  # Do not list
5969     _VALID_URL = r'''(?x)
5970         (?:https?://)?
5971         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5972         (?:watch\?(?:
5973             feature=[a-z_]+|
5974             annotation_id=annotation_[^&]+|
5975             x-yt-cl=[0-9]+|
5976             hl=[^&]*|
5977             t=[0-9]+
5978         )?
5979         |
5980             attribution_link\?a=[^&]+
5981         )
5982         $
5983     '''
5984
5985     _TESTS = [{
5986         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5987         'only_matching': True,
5988     }, {
5989         'url': 'https://www.youtube.com/watch?',
5990         'only_matching': True,
5991     }, {
5992         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5993         'only_matching': True,
5994     }, {
5995         'url': 'https://www.youtube.com/watch?feature=foo',
5996         'only_matching': True,
5997     }, {
5998         'url': 'https://www.youtube.com/watch?hl=en-GB',
5999         'only_matching': True,
6000     }, {
6001         'url': 'https://www.youtube.com/watch?t=2372',
6002         'only_matching': True,
6003     }]
6004
6005     def _real_extract(self, url):
6006         raise ExtractorError(
6007             'Did you forget to quote the URL? Remember that & is a meta '
6008             'character in most shells, so you want to put the URL in quotes, '
6009             'like  youtube-dl '
6010             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6011             ' or simply  youtube-dl BaW_jenozKc  .',
6012             expected=True)
6013
6014
6015 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6016     IE_NAME = 'youtube:clip'
6017     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6018     _TESTS = [{
6019         # FIXME: Other metadata should be extracted from the clip, not from the base video
6020         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6021         'info_dict': {
6022             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6023             'ext': 'mp4',
6024             'section_start': 29.0,
6025             'section_end': 39.7,
6026             'duration': 10.7,
6027             'age_limit': 0,
6028             'availability': 'public',
6029             'categories': ['Gaming'],
6030             'channel': 'Scott The Woz',
6031             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6032             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6033             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6034             'like_count': int,
6035             'playable_in_embed': True,
6036             'tags': 'count:17',
6037             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6038             'title': 'Mobile Games on Console - Scott The Woz',
6039             'upload_date': '20210920',
6040             'uploader': 'Scott The Woz',
6041             'uploader_id': 'scottthewoz',
6042             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6043             'view_count': int,
6044             'live_status': 'not_live',
6045             'channel_follower_count': int
6046         }
6047     }]
6048
6049     def _real_extract(self, url):
6050         clip_id = self._match_id(url)
6051         _, data = self._extract_webpage(url, clip_id)
6052
6053         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6054         if not video_id:
6055             raise ExtractorError('Unable to find video ID')
6056
6057         clip_data = traverse_obj(data, (
6058             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6059             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6060             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6061             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6062
6063         return {
6064             '_type': 'url_transparent',
6065             'url': f'https://www.youtube.com/watch?v={video_id}',
6066             'ie_key': YoutubeIE.ie_key(),
6067             'id': clip_id,
6068             'section_start': int(clip_data['startTimeMs']) / 1000,
6069             'section_end': int(clip_data['endTimeMs']) / 1000,
6070         }
6071
6072
6073 class YoutubeTruncatedIDIE(InfoExtractor):
6074     IE_NAME = 'youtube:truncated_id'
6075     IE_DESC = False  # Do not list
6076     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6077
6078     _TESTS = [{
6079         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6080         'only_matching': True,
6081     }]
6082
6083     def _real_extract(self, url):
6084         video_id = self._match_id(url)
6085         raise ExtractorError(
6086             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6087             expected=True)