yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16 import urllib.error
  17 import urllib.parse
  18
  19 from .common import InfoExtractor, SearchInfoExtractor
  20 from ..compat import functools
  21 from ..jsinterp import JSInterpreter
  22 from ..utils import (
  23     NO_DEFAULT,
  24     ExtractorError,
  25     UserNotLive,
  26     bug_reports_message,
  27     classproperty,
  28     clean_html,
  29     datetime_from_str,
  30     dict_get,
  31     error_to_compat_str,
  32     float_or_none,
  33     format_field,
  34     get_first,
  35     int_or_none,
  36     is_html,
  37     join_nonempty,
  38     js_to_json,
  39     mimetype2ext,
  40     network_exceptions,
  41     orderedSet,
  42     parse_codecs,
  43     parse_count,
  44     parse_duration,
  45     parse_iso8601,
  46     parse_qs,
  47     qualities,
  48     remove_end,
  49     remove_start,
  50     smuggle_url,
  51     str_or_none,
  52     str_to_int,
  53     strftime_or_none,
  54     traverse_obj,
  55     try_get,
  56     unescapeHTML,
  57     unified_strdate,
  58     unified_timestamp,
  59     unsmuggle_url,
  60     update_url_query,
  61     url_or_none,
  62     urljoin,
  63     variadic,
  64 )
  65
  66 # any clients starting with _ cannot be explicitly requested by the user
  67 INNERTUBE_CLIENTS = {
  68     'web': {
  69         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  70         'INNERTUBE_CONTEXT': {
  71             'client': {
  72                 'clientName': 'WEB',
  73                 'clientVersion': '2.20211221.00.00',
  74             }
  75         },
  76         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  77     },
  78     'web_embedded': {
  79         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  80         'INNERTUBE_CONTEXT': {
  81             'client': {
  82                 'clientName': 'WEB_EMBEDDED_PLAYER',
  83                 'clientVersion': '1.20211215.00.01',
  84             },
  85         },
  86         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  87     },
  88     'web_music': {
  89         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  90         'INNERTUBE_HOST': 'music.youtube.com',
  91         'INNERTUBE_CONTEXT': {
  92             'client': {
  93                 'clientName': 'WEB_REMIX',
  94                 'clientVersion': '1.20211213.00.00',
  95             }
  96         },
  97         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
  98     },
  99     'web_creator': {
 100         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 101         'INNERTUBE_CONTEXT': {
 102             'client': {
 103                 'clientName': 'WEB_CREATOR',
 104                 'clientVersion': '1.20211220.02.00',
 105             }
 106         },
 107         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 108     },
 109     'android': {
 110         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 111         'INNERTUBE_CONTEXT': {
 112             'client': {
 113                 'clientName': 'ANDROID',
 114                 'clientVersion': '16.49',
 115             }
 116         },
 117         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 118         'REQUIRE_JS_PLAYER': False
 119     },
 120     'android_embedded': {
 121         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 122         'INNERTUBE_CONTEXT': {
 123             'client': {
 124                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 125                 'clientVersion': '16.49',
 126             },
 127         },
 128         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 129         'REQUIRE_JS_PLAYER': False
 130     },
 131     'android_music': {
 132         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 133         'INNERTUBE_CONTEXT': {
 134             'client': {
 135                 'clientName': 'ANDROID_MUSIC',
 136                 'clientVersion': '4.57',
 137             }
 138         },
 139         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 140         'REQUIRE_JS_PLAYER': False
 141     },
 142     'android_creator': {
 143         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 144         'INNERTUBE_CONTEXT': {
 145             'client': {
 146                 'clientName': 'ANDROID_CREATOR',
 147                 'clientVersion': '21.47',
 148             },
 149         },
 150         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 151         'REQUIRE_JS_PLAYER': False
 152     },
 153     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 154     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 155     'ios': {
 156         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 157         'INNERTUBE_CONTEXT': {
 158             'client': {
 159                 'clientName': 'IOS',
 160                 'clientVersion': '16.46',
 161                 'deviceModel': 'iPhone14,3',
 162             }
 163         },
 164         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 165         'REQUIRE_JS_PLAYER': False
 166     },
 167     'ios_embedded': {
 168         'INNERTUBE_CONTEXT': {
 169             'client': {
 170                 'clientName': 'IOS_MESSAGES_EXTENSION',
 171                 'clientVersion': '16.46',
 172                 'deviceModel': 'iPhone14,3',
 173             },
 174         },
 175         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 176         'REQUIRE_JS_PLAYER': False
 177     },
 178     'ios_music': {
 179         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 180         'INNERTUBE_CONTEXT': {
 181             'client': {
 182                 'clientName': 'IOS_MUSIC',
 183                 'clientVersion': '4.57',
 184             },
 185         },
 186         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 187         'REQUIRE_JS_PLAYER': False
 188     },
 189     'ios_creator': {
 190         'INNERTUBE_CONTEXT': {
 191             'client': {
 192                 'clientName': 'IOS_CREATOR',
 193                 'clientVersion': '21.47',
 194             },
 195         },
 196         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 197         'REQUIRE_JS_PLAYER': False
 198     },
 199     # mweb has 'ultralow' formats
 200     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 201     'mweb': {
 202         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 203         'INNERTUBE_CONTEXT': {
 204             'client': {
 205                 'clientName': 'MWEB',
 206                 'clientVersion': '2.20211221.01.00',
 207             }
 208         },
 209         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 210     },
 211     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 212     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 213     'tv_embedded': {
 214         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 215         'INNERTUBE_CONTEXT': {
 216             'client': {
 217                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 218                 'clientVersion': '2.0',
 219             },
 220         },
 221         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 222     },
 223 }
 224
 225
 226 def _split_innertube_client(client_name):
 227     variant, *base = client_name.rsplit('.', 1)
 228     if base:
 229         return variant, base[0], variant
 230     base, *variant = client_name.split('_', 1)
 231     return client_name, base, variant[0] if variant else None
 232
 233
 234 def build_innertube_clients():
 235     THIRD_PARTY = {
 236         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 237     }
 238     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 239     priority = qualities(BASE_CLIENTS[::-1])
 240
 241     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 242         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 243         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 244         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 245         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 246
 247         _, base_client, variant = _split_innertube_client(client)
 248         ytcfg['priority'] = 10 * priority(base_client)
 249
 250         if not variant:
 251             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 252             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 253             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 254             embedscreen['priority'] -= 3
 255         elif variant == 'embedded':
 256             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 257             ytcfg['priority'] -= 2
 258         else:
 259             ytcfg['priority'] -= 3
 260
 261
 262 build_innertube_clients()
 263
 264
 265 class YoutubeBaseInfoExtractor(InfoExtractor):
 266     """Provide base functions for Youtube extractors"""
 267
 268     _RESERVED_NAMES = (
 269         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 270         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 271         r'browse|oembed|get_video_info|iframe_api|s/player|'
 272         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 273
 274     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 275
 276     # _NETRC_MACHINE = 'youtube'
 277
 278     # If True it will raise an error if no login info is provided
 279     _LOGIN_REQUIRED = False
 280
 281     _INVIDIOUS_SITES = (
 282         # invidious-redirect websites
 283         r'(?:www\.)?redirect\.invidious\.io',
 284         r'(?:(?:www|dev)\.)?invidio\.us',
 285         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 286         r'(?:www\.)?invidious\.pussthecat\.org',
 287         r'(?:www\.)?invidious\.zee\.li',
 288         r'(?:www\.)?invidious\.ethibox\.fr',
 289         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 290         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 291         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 292         # youtube-dl invidious instances list
 293         r'(?:(?:www|no)\.)?invidiou\.sh',
 294         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 295         r'(?:www\.)?invidious\.kabi\.tk',
 296         r'(?:www\.)?invidious\.mastodon\.host',
 297         r'(?:www\.)?invidious\.zapashcanon\.fr',
 298         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 299         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 300         r'(?:www\.)?invidious\.himiko\.cloud',
 301         r'(?:www\.)?invidious\.reallyancient\.tech',
 302         r'(?:www\.)?invidious\.tube',
 303         r'(?:www\.)?invidiou\.site',
 304         r'(?:www\.)?invidious\.site',
 305         r'(?:www\.)?invidious\.xyz',
 306         r'(?:www\.)?invidious\.nixnet\.xyz',
 307         r'(?:www\.)?invidious\.048596\.xyz',
 308         r'(?:www\.)?invidious\.drycat\.fr',
 309         r'(?:www\.)?inv\.skyn3t\.in',
 310         r'(?:www\.)?tube\.poal\.co',
 311         r'(?:www\.)?tube\.connect\.cafe',
 312         r'(?:www\.)?vid\.wxzm\.sx',
 313         r'(?:www\.)?vid\.mint\.lgbt',
 314         r'(?:www\.)?vid\.puffyan\.us',
 315         r'(?:www\.)?yewtu\.be',
 316         r'(?:www\.)?yt\.elukerio\.org',
 317         r'(?:www\.)?yt\.lelux\.fi',
 318         r'(?:www\.)?invidious\.ggc-project\.de',
 319         r'(?:www\.)?yt\.maisputain\.ovh',
 320         r'(?:www\.)?ytprivate\.com',
 321         r'(?:www\.)?invidious\.13ad\.de',
 322         r'(?:www\.)?invidious\.toot\.koeln',
 323         r'(?:www\.)?invidious\.fdn\.fr',
 324         r'(?:www\.)?watch\.nettohikari\.com',
 325         r'(?:www\.)?invidious\.namazso\.eu',
 326         r'(?:www\.)?invidious\.silkky\.cloud',
 327         r'(?:www\.)?invidious\.exonip\.de',
 328         r'(?:www\.)?invidious\.riverside\.rocks',
 329         r'(?:www\.)?invidious\.blamefran\.net',
 330         r'(?:www\.)?invidious\.moomoo\.de',
 331         r'(?:www\.)?ytb\.trom\.tf',
 332         r'(?:www\.)?yt\.cyberhost\.uk',
 333         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 334         r'(?:www\.)?qklhadlycap4cnod\.onion',
 335         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 336         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 337         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 338         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 339         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 340         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 341         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 342         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 343         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 344         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 345         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 346         r'(?:www\.)?piped\.kavin\.rocks',
 347         r'(?:www\.)?piped\.silkky\.cloud',
 348         r'(?:www\.)?piped\.tokhmi\.xyz',
 349         r'(?:www\.)?piped\.moomoo\.me',
 350         r'(?:www\.)?il\.ax',
 351         r'(?:www\.)?piped\.syncpundit\.com',
 352         r'(?:www\.)?piped\.mha\.fi',
 353         r'(?:www\.)?piped\.mint\.lgbt',
 354         r'(?:www\.)?piped\.privacy\.com\.de',
 355     )
 356
 357     def _initialize_consent(self):
 358         cookies = self._get_cookies('https://www.youtube.com/')
 359         if cookies.get('__Secure-3PSID'):
 360             return
 361         consent_id = None
 362         consent = cookies.get('CONSENT')
 363         if consent:
 364             if 'YES' in consent.value:
 365                 return
 366             consent_id = self._search_regex(
 367                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 368         if not consent_id:
 369             consent_id = random.randint(100, 999)
 370         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 371
 372     def _initialize_pref(self):
 373         cookies = self._get_cookies('https://www.youtube.com/')
 374         pref_cookie = cookies.get('PREF')
 375         pref = {}
 376         if pref_cookie:
 377             try:
 378                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 379             except ValueError:
 380                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 381         pref.update({'hl': 'en', 'tz': 'UTC'})
 382         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 383
 384     def _real_initialize(self):
 385         self._initialize_pref()
 386         self._initialize_consent()
 387         self._check_login_required()
 388
 389     def _check_login_required(self):
 390         if self._LOGIN_REQUIRED and not self._cookies_passed:
 391             self.raise_login_required('Login details are needed to download this content', method='cookies')
 392
 393     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 394     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 395
 396     def _get_default_ytcfg(self, client='web'):
 397         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 398
 399     def _get_innertube_host(self, client='web'):
 400         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 401
 402     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 403         # try_get but with fallback to default ytcfg client values when present
 404         _func = lambda y: try_get(y, getter, expected_type)
 405         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 406
 407     def _extract_client_name(self, ytcfg, default_client='web'):
 408         return self._ytcfg_get_safe(
 409             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 410                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 411
 412     def _extract_client_version(self, ytcfg, default_client='web'):
 413         return self._ytcfg_get_safe(
 414             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 415                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 416
 417     def _select_api_hostname(self, req_api_hostname, default_client=None):
 418         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 419                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 420
 421     def _extract_api_key(self, ytcfg=None, default_client='web'):
 422         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 423
 424     def _extract_context(self, ytcfg=None, default_client='web'):
 425         context = get_first(
 426             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 427         # Enforce language and tz for extraction
 428         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 429         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 430         return context
 431
 432     _SAPISID = None
 433
 434     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 435         time_now = round(time.time())
 436         if self._SAPISID is None:
 437             yt_cookies = self._get_cookies('https://www.youtube.com')
 438             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 439             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 440             sapisid_cookie = dict_get(
 441                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 442             if sapisid_cookie and sapisid_cookie.value:
 443                 self._SAPISID = sapisid_cookie.value
 444                 self.write_debug('Extracted SAPISID cookie')
 445                 # SAPISID cookie is required if not already present
 446                 if not yt_cookies.get('SAPISID'):
 447                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 448                     self._set_cookie(
 449                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 450             else:
 451                 self._SAPISID = False
 452         if not self._SAPISID:
 453             return None
 454         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 455         sapisidhash = hashlib.sha1(
 456             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 457         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 458
 459     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 460                   note='Downloading API JSON', errnote='Unable to download API page',
 461                   context=None, api_key=None, api_hostname=None, default_client='web'):
 462
 463         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 464         data.update(query)
 465         real_headers = self.generate_api_headers(default_client=default_client)
 466         real_headers.update({'content-type': 'application/json'})
 467         if headers:
 468             real_headers.update(headers)
 469         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 470                    or api_key or self._extract_api_key(default_client=default_client))
 471         return self._download_json(
 472             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 473             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 474             data=json.dumps(data).encode('utf8'), headers=real_headers,
 475             query={'key': api_key, 'prettyPrint': 'false'})
 476
 477     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 478         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 479
 480     @staticmethod
 481     def _extract_session_index(*data):
 482         """
 483         Index of current account in account list.
 484         See: https://github.com/yt-dlp/yt-dlp/pull/519
 485         """
 486         for ytcfg in data:
 487             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 488             if session_index is not None:
 489                 return session_index
 490
 491     # Deprecated?
 492     def _extract_identity_token(self, ytcfg=None, webpage=None):
 493         if ytcfg:
 494             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 495             if token:
 496                 return token
 497         if webpage:
 498             return self._search_regex(
 499                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 500                 'identity token', default=None, fatal=False)
 501
 502     @staticmethod
 503     def _extract_account_syncid(*args):
 504         """
 505         Extract syncId required to download private playlists of secondary channels
 506         @params response and/or ytcfg
 507         """
 508         for data in args:
 509             # ytcfg includes channel_syncid if on secondary channel
 510             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 511             if delegated_sid:
 512                 return delegated_sid
 513             sync_ids = (try_get(
 514                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 515                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 516             if len(sync_ids) >= 2 and sync_ids[1]:
 517                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 518                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 519                 return sync_ids[0]
 520
 521     @staticmethod
 522     def _extract_visitor_data(*args):
 523         """
 524         Extracts visitorData from an API response or ytcfg
 525         Appears to be used to track session state
 526         """
 527         return get_first(
 528             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 529             expected_type=str)
 530
 531     @functools.cached_property
 532     def is_authenticated(self):
 533         return bool(self._generate_sapisidhash_header())
 534
 535     def extract_ytcfg(self, video_id, webpage):
 536         if not webpage:
 537             return {}
 538         return self._parse_json(
 539             self._search_regex(
 540                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 541                 default='{}'), video_id, fatal=False) or {}
 542
 543     def generate_api_headers(
 544             self, *, ytcfg=None, account_syncid=None, session_index=None,
 545             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 546
 547         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 548         headers = {
 549             'X-YouTube-Client-Name': str(
 550                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 551             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 552             'Origin': origin,
 553             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 554             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 555             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 556         }
 557         if session_index is None:
 558             session_index = self._extract_session_index(ytcfg)
 559         if account_syncid or session_index is not None:
 560             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 561
 562         auth = self._generate_sapisidhash_header(origin)
 563         if auth is not None:
 564             headers['Authorization'] = auth
 565             headers['X-Origin'] = origin
 566         return {h: v for h, v in headers.items() if v is not None}
 567
 568     def _download_ytcfg(self, client, video_id):
 569         url = {
 570             'web': 'https://www.youtube.com',
 571             'web_music': 'https://music.youtube.com',
 572             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 573         }.get(client)
 574         if not url:
 575             return {}
 576         webpage = self._download_webpage(
 577             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 578         return self.extract_ytcfg(video_id, webpage) or {}
 579
 580     @staticmethod
 581     def _build_api_continuation_query(continuation, ctp=None):
 582         query = {
 583             'continuation': continuation
 584         }
 585         # TODO: Inconsistency with clickTrackingParams.
 586         # Currently we have a fixed ctp contained within context (from ytcfg)
 587         # and a ctp in root query for continuation.
 588         if ctp:
 589             query['clickTracking'] = {'clickTrackingParams': ctp}
 590         return query
 591
 592     @classmethod
 593     def _extract_next_continuation_data(cls, renderer):
 594         next_continuation = try_get(
 595             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 596                        lambda x: x['continuation']['reloadContinuationData']), dict)
 597         if not next_continuation:
 598             return
 599         continuation = next_continuation.get('continuation')
 600         if not continuation:
 601             return
 602         ctp = next_continuation.get('clickTrackingParams')
 603         return cls._build_api_continuation_query(continuation, ctp)
 604
 605     @classmethod
 606     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 607         if isinstance(continuation_ep, dict):
 608             continuation = try_get(
 609                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 610             if not continuation:
 611                 return
 612             ctp = continuation_ep.get('clickTrackingParams')
 613             return cls._build_api_continuation_query(continuation, ctp)
 614
 615     @classmethod
 616     def _extract_continuation(cls, renderer):
 617         next_continuation = cls._extract_next_continuation_data(renderer)
 618         if next_continuation:
 619             return next_continuation
 620
 621         contents = []
 622         for key in ('contents', 'items'):
 623             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 624
 625         for content in contents:
 626             if not isinstance(content, dict):
 627                 continue
 628             continuation_ep = try_get(
 629                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 630                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 631                 dict)
 632             continuation = cls._extract_continuation_ep_data(continuation_ep)
 633             if continuation:
 634                 return continuation
 635
 636     @classmethod
 637     def _extract_alerts(cls, data):
 638         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 639             if not isinstance(alert_dict, dict):
 640                 continue
 641             for alert in alert_dict.values():
 642                 alert_type = alert.get('type')
 643                 if not alert_type:
 644                     continue
 645                 message = cls._get_text(alert, 'text')
 646                 if message:
 647                     yield alert_type, message
 648
 649     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 650         errors = []
 651         warnings = []
 652         for alert_type, alert_message in alerts:
 653             if alert_type.lower() == 'error' and fatal:
 654                 errors.append([alert_type, alert_message])
 655             else:
 656                 warnings.append([alert_type, alert_message])
 657
 658         for alert_type, alert_message in (warnings + errors[:-1]):
 659             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 660         if errors:
 661             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 662
 663     def _extract_and_report_alerts(self, data, *args, **kwargs):
 664         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 665
 666     def _extract_badges(self, renderer: dict):
 667         badges = set()
 668         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 669             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], str)
 670             if label:
 671                 badges.add(label.lower())
 672         return badges
 673
 674     @staticmethod
 675     def _get_text(data, *path_list, max_runs=None):
 676         for path in path_list or [None]:
 677             if path is None:
 678                 obj = [data]
 679             else:
 680                 obj = traverse_obj(data, path, default=[])
 681                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 682                     obj = [obj]
 683             for item in obj:
 684                 text = try_get(item, lambda x: x['simpleText'], str)
 685                 if text:
 686                     return text
 687                 runs = try_get(item, lambda x: x['runs'], list) or []
 688                 if not runs and isinstance(item, list):
 689                     runs = item
 690
 691                 runs = runs[:min(len(runs), max_runs or len(runs))]
 692                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 693                 if text:
 694                     return text
 695
 696     def _get_count(self, data, *path_list):
 697         count_text = self._get_text(data, *path_list) or ''
 698         count = parse_count(count_text)
 699         if count is None:
 700             count = str_to_int(
 701                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 702         return count
 703
 704     @staticmethod
 705     def _extract_thumbnails(data, *path_list):
 706         """
 707         Extract thumbnails from thumbnails dict
 708         @param path_list: path list to level that contains 'thumbnails' key
 709         """
 710         thumbnails = []
 711         for path in path_list or [()]:
 712             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 713                 thumbnail_url = url_or_none(thumbnail.get('url'))
 714                 if not thumbnail_url:
 715                     continue
 716                 # Sometimes youtube gives a wrong thumbnail URL. See:
 717                 # https://github.com/yt-dlp/yt-dlp/issues/233
 718                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 719                 if 'maxresdefault' in thumbnail_url:
 720                     thumbnail_url = thumbnail_url.split('?')[0]
 721                 thumbnails.append({
 722                     'url': thumbnail_url,
 723                     'height': int_or_none(thumbnail.get('height')),
 724                     'width': int_or_none(thumbnail.get('width')),
 725                 })
 726         return thumbnails
 727
 728     @staticmethod
 729     def extract_relative_time(relative_time_text):
 730         """
 731         Extracts a relative time from string and converts to dt object
 732         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 733         """
 734         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 735         if mobj:
 736             start = mobj.group('start')
 737             if start:
 738                 return datetime_from_str(start)
 739             try:
 740                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 741             except ValueError:
 742                 return None
 743
 744     def _extract_time_text(self, renderer, *path_list):
 745         """@returns (timestamp, time_text)"""
 746         text = self._get_text(renderer, *path_list) or ''
 747         dt = self.extract_relative_time(text)
 748         timestamp = None
 749         if isinstance(dt, datetime.datetime):
 750             timestamp = calendar.timegm(dt.timetuple())
 751
 752         if timestamp is None:
 753             timestamp = (
 754                 unified_timestamp(text) or unified_timestamp(
 755                     self._search_regex(
 756                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 757                         text.lower(), 'time text', default=None)))
 758
 759         if text and timestamp is None:
 760             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 761         return timestamp, text
 762
 763     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 764                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 765                           default_client='web'):
 766         response = None
 767         last_error = None
 768         count = -1
 769         retries = self.get_param('extractor_retries', 3)
 770         if check_get_keys is None:
 771             check_get_keys = []
 772         while count < retries:
 773             count += 1
 774             if last_error:
 775                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 776             try:
 777                 response = self._call_api(
 778                     ep=ep, fatal=True, headers=headers,
 779                     video_id=item_id, query=query,
 780                     context=self._extract_context(ytcfg, default_client),
 781                     api_key=self._extract_api_key(ytcfg, default_client),
 782                     api_hostname=api_hostname, default_client=default_client,
 783                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 784             except ExtractorError as e:
 785                 if isinstance(e.cause, network_exceptions):
 786                     if isinstance(e.cause, urllib.error.HTTPError):
 787                         first_bytes = e.cause.read(512)
 788                         if not is_html(first_bytes):
 789                             yt_error = try_get(
 790                                 self._parse_json(
 791                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 792                                 lambda x: x['error']['message'], str)
 793                             if yt_error:
 794                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 795                     # Downloading page may result in intermittent 5xx HTTP error
 796                     # Sometimes a 404 is also received. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 797                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 798                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 799                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
 800                         last_error = error_to_compat_str(e.cause or e.msg)
 801                         if count < retries:
 802                             continue
 803                 if fatal:
 804                     raise
 805                 else:
 806                     self.report_warning(error_to_compat_str(e))
 807                     return
 808
 809             else:
 810                 try:
 811                     self._extract_and_report_alerts(response, only_once=True)
 812                 except ExtractorError as e:
 813                     # YouTube servers may return errors we want to retry on in a 200 OK response
 814                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 815                     if 'unknown error' in e.msg.lower():
 816                         last_error = e.msg
 817                         continue
 818                     if fatal:
 819                         raise
 820                     self.report_warning(error_to_compat_str(e))
 821                     return
 822                 if not check_get_keys or dict_get(response, check_get_keys):
 823                     break
 824                 # Youtube sometimes sends incomplete data
 825                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 826                 last_error = 'Incomplete data received'
 827                 if count >= retries:
 828                     if fatal:
 829                         raise ExtractorError(last_error)
 830                     else:
 831                         self.report_warning(last_error)
 832                         return
 833         return response
 834
 835     @staticmethod
 836     def is_music_url(url):
 837         return re.match(r'https?://music\.youtube\.com/', url) is not None
 838
 839     def _extract_video(self, renderer):
 840         video_id = renderer.get('videoId')
 841         title = self._get_text(renderer, 'title')
 842         description = self._get_text(renderer, 'descriptionSnippet')
 843         duration = parse_duration(self._get_text(
 844             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 845         if duration is None:
 846             duration = parse_duration(self._search_regex(
 847                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 848                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 849                 video_id, default=None, group='duration'))
 850
 851         view_count = self._get_count(renderer, 'viewCountText')
 852
 853         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 854         channel_id = traverse_obj(
 855             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 856             expected_type=str, get_all=False)
 857         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 858         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 859         overlay_style = traverse_obj(
 860             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 861             get_all=False, expected_type=str)
 862         badges = self._extract_badges(renderer)
 863         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 864         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 865             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 866             expected_type=str)) or ''
 867         url = f'https://www.youtube.com/watch?v={video_id}'
 868         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 869             url = f'https://www.youtube.com/shorts/{video_id}'
 870
 871         return {
 872             '_type': 'url',
 873             'ie_key': YoutubeIE.ie_key(),
 874             'id': video_id,
 875             'url': url,
 876             'title': title,
 877             'description': description,
 878             'duration': duration,
 879             'view_count': view_count,
 880             'uploader': uploader,
 881             'channel_id': channel_id,
 882             'thumbnails': thumbnails,
 883             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 884                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 885                             else None),
 886             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 887                             else 'was_live' if 'streamed' in time_text.lower()
 888                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 889                             else None),
 890             'release_timestamp': scheduled_timestamp,
 891             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 892         }
 893
 894
 895 class YoutubeIE(YoutubeBaseInfoExtractor):
 896     IE_DESC = 'YouTube'
 897     _VALID_URL = r"""(?x)^
 898                      (
 899                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 900                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 901                             (?:www\.)?deturl\.com/www\.youtube\.com|
 902                             (?:www\.)?pwnyoutube\.com|
 903                             (?:www\.)?hooktube\.com|
 904                             (?:www\.)?yourepeat\.com|
 905                             tube\.majestyc\.net|
 906                             %(invidious)s|
 907                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 908                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 909                          (?:                                                  # the various things that can precede the ID:
 910                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 911                              |(?:                                             # or the v= param in all its forms
 912                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 913                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 914                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 915                                  v=
 916                              )
 917                          ))
 918                          |(?:
 919                             youtu\.be|                                        # just youtu.be/xxxx
 920                             vid\.plus|                                        # or vid.plus/xxxx
 921                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 922                             %(invidious)s
 923                          )/
 924                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 925                          )
 926                      )?                                                       # all until now is optional -> you can pass the naked ID
 927                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 928                      (?(1).+)?                                                # if we found the ID, everything can follow
 929                      (?:\#|$)""" % {
 930         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 931     }
 932     _PLAYER_INFO_RE = (
 933         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 934         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 935         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 936     )
 937     _formats = {
 938         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 939         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 940         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 941         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 942         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 943         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 944         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 945         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 946         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 947         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 948         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 949         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 950         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 951         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 952         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 953         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 954         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 955         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 956
 957
 958         # 3D videos
 959         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 960         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 961         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 962         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 963         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 964         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 965         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 966
 967         # Apple HTTP Live Streaming
 968         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 969         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 970         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 971         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 972         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 973         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 974         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 975         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 976
 977         # DASH mp4 video
 978         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 979         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 980         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 981         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 982         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 983         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 984         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 985         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 986         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 988         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 989         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 990
 991         # Dash mp4 audio
 992         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 993         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 994         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 995         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 996         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 997         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 998         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 999
1000         # Dash webm
1001         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1002         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1003         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1004         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1005         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1006         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1007         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1008         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1009         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1010         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1011         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1012         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1013         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1014         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1015         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1016         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1017         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1018         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1019         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1020         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1021         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1022         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1023
1024         # Dash webm audio
1025         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1026         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1027
1028         # Dash webm audio with opus inside
1029         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1030         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1031         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1032
1033         # RTMP (unnamed)
1034         '_rtmp': {'protocol': 'rtmp'},
1035
1036         # av01 video only formats sometimes served with "unknown" codecs
1037         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1038         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1039         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1040         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1041         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1042         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1043         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1044         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1045     }
1046     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1047
1048     _GEO_BYPASS = False
1049
1050     IE_NAME = 'youtube'
1051     _TESTS = [
1052         {
1053             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1054             'info_dict': {
1055                 'id': 'BaW_jenozKc',
1056                 'ext': 'mp4',
1057                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1058                 'uploader': 'Philipp Hagemeister',
1059                 'uploader_id': 'phihag',
1060                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1061                 'channel': 'Philipp Hagemeister',
1062                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1063                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1064                 'upload_date': '20121002',
1065                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1066                 'categories': ['Science & Technology'],
1067                 'tags': ['youtube-dl'],
1068                 'duration': 10,
1069                 'view_count': int,
1070                 'like_count': int,
1071                 'availability': 'public',
1072                 'playable_in_embed': True,
1073                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1074                 'live_status': 'not_live',
1075                 'age_limit': 0,
1076                 'start_time': 1,
1077                 'end_time': 9,
1078                 'comment_count': int,
1079                 'channel_follower_count': int
1080             }
1081         },
1082         {
1083             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1084             'note': 'Embed-only video (#1746)',
1085             'info_dict': {
1086                 'id': 'yZIXLfi8CZQ',
1087                 'ext': 'mp4',
1088                 'upload_date': '20120608',
1089                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1090                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1091                 'uploader': 'SET India',
1092                 'uploader_id': 'setindia',
1093                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1094                 'age_limit': 18,
1095             },
1096             'skip': 'Private video',
1097         },
1098         {
1099             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1100             'note': 'Use the first video ID in the URL',
1101             'info_dict': {
1102                 'id': 'BaW_jenozKc',
1103                 'ext': 'mp4',
1104                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1105                 'uploader': 'Philipp Hagemeister',
1106                 'uploader_id': 'phihag',
1107                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1108                 'channel': 'Philipp Hagemeister',
1109                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1110                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1111                 'upload_date': '20121002',
1112                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1113                 'categories': ['Science & Technology'],
1114                 'tags': ['youtube-dl'],
1115                 'duration': 10,
1116                 'view_count': int,
1117                 'like_count': int,
1118                 'availability': 'public',
1119                 'playable_in_embed': True,
1120                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1121                 'live_status': 'not_live',
1122                 'age_limit': 0,
1123                 'comment_count': int,
1124                 'channel_follower_count': int
1125             },
1126             'params': {
1127                 'skip_download': True,
1128             },
1129         },
1130         {
1131             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1132             'note': '256k DASH audio (format 141) via DASH manifest',
1133             'info_dict': {
1134                 'id': 'a9LDPn-MO4I',
1135                 'ext': 'm4a',
1136                 'upload_date': '20121002',
1137                 'uploader_id': '8KVIDEO',
1138                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1139                 'description': '',
1140                 'uploader': '8KVIDEO',
1141                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1142             },
1143             'params': {
1144                 'youtube_include_dash_manifest': True,
1145                 'format': '141',
1146             },
1147             'skip': 'format 141 not served anymore',
1148         },
1149         # DASH manifest with encrypted signature
1150         {
1151             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1152             'info_dict': {
1153                 'id': 'IB3lcPjvWLA',
1154                 'ext': 'm4a',
1155                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1156                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1157                 'duration': 244,
1158                 'uploader': 'AfrojackVEVO',
1159                 'uploader_id': 'AfrojackVEVO',
1160                 'upload_date': '20131011',
1161                 'abr': 129.495,
1162                 'like_count': int,
1163                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1164                 'playable_in_embed': True,
1165                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1166                 'view_count': int,
1167                 'track': 'The Spark',
1168                 'live_status': 'not_live',
1169                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1170                 'channel': 'Afrojack',
1171                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1172                 'tags': 'count:19',
1173                 'availability': 'public',
1174                 'categories': ['Music'],
1175                 'age_limit': 0,
1176                 'alt_title': 'The Spark',
1177                 'channel_follower_count': int
1178             },
1179             'params': {
1180                 'youtube_include_dash_manifest': True,
1181                 'format': '141/bestaudio[ext=m4a]',
1182             },
1183         },
1184         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1185         {
1186             'note': 'Embed allowed age-gate video',
1187             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1188             'info_dict': {
1189                 'id': 'HtVdAasjOgU',
1190                 'ext': 'mp4',
1191                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1192                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1193                 'duration': 142,
1194                 'uploader': 'The Witcher',
1195                 'uploader_id': 'WitcherGame',
1196                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1197                 'upload_date': '20140605',
1198                 'age_limit': 18,
1199                 'categories': ['Gaming'],
1200                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1201                 'availability': 'needs_auth',
1202                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1203                 'like_count': int,
1204                 'channel': 'The Witcher',
1205                 'live_status': 'not_live',
1206                 'tags': 'count:17',
1207                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1208                 'playable_in_embed': True,
1209                 'view_count': int,
1210                 'channel_follower_count': int
1211             },
1212         },
1213         {
1214             'note': 'Age-gate video with embed allowed in public site',
1215             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1216             'info_dict': {
1217                 'id': 'HsUATh_Nc2U',
1218                 'ext': 'mp4',
1219                 'title': 'Godzilla 2 (Official Video)',
1220                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1221                 'upload_date': '20200408',
1222                 'uploader_id': 'FlyingKitty900',
1223                 'uploader': 'FlyingKitty',
1224                 'age_limit': 18,
1225                 'availability': 'needs_auth',
1226                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1227                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1228                 'channel': 'FlyingKitty',
1229                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1230                 'view_count': int,
1231                 'categories': ['Entertainment'],
1232                 'live_status': 'not_live',
1233                 'tags': ['Flyingkitty', 'godzilla 2'],
1234                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1235                 'like_count': int,
1236                 'duration': 177,
1237                 'playable_in_embed': True,
1238                 'channel_follower_count': int
1239             },
1240         },
1241         {
1242             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1243             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1244             'info_dict': {
1245                 'id': 'Tq92D6wQ1mg',
1246                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1247                 'ext': 'mp4',
1248                 'upload_date': '20191228',
1249                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1250                 'uploader': 'Projekt Melody',
1251                 'description': 'md5:17eccca93a786d51bc67646756894066',
1252                 'age_limit': 18,
1253                 'like_count': int,
1254                 'availability': 'needs_auth',
1255                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1256                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1257                 'view_count': int,
1258                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1259                 'channel': 'Projekt Melody',
1260                 'live_status': 'not_live',
1261                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1262                 'playable_in_embed': True,
1263                 'categories': ['Entertainment'],
1264                 'duration': 106,
1265                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1266                 'comment_count': int,
1267                 'channel_follower_count': int
1268             },
1269         },
1270         {
1271             'note': 'Non-Agegated non-embeddable video',
1272             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1273             'info_dict': {
1274                 'id': 'MeJVWBSsPAY',
1275                 'ext': 'mp4',
1276                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1277                 'uploader': 'Herr Lurik',
1278                 'uploader_id': 'st3in234',
1279                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1280                 'upload_date': '20130730',
1281                 'track': 'Such mich find mich',
1282                 'age_limit': 0,
1283                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1284                 'like_count': int,
1285                 'playable_in_embed': False,
1286                 'creator': 'OOMPH!',
1287                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1288                 'view_count': int,
1289                 'alt_title': 'Such mich find mich',
1290                 'duration': 210,
1291                 'channel': 'Herr Lurik',
1292                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1293                 'categories': ['Music'],
1294                 'availability': 'public',
1295                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1296                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1297                 'live_status': 'not_live',
1298                 'artist': 'OOMPH!',
1299                 'channel_follower_count': int
1300             },
1301         },
1302         {
1303             'note': 'Non-bypassable age-gated video',
1304             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1305             'only_matching': True,
1306         },
1307         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1308         # YouTube Red ad is not captured for creator
1309         {
1310             'url': '__2ABJjxzNo',
1311             'info_dict': {
1312                 'id': '__2ABJjxzNo',
1313                 'ext': 'mp4',
1314                 'duration': 266,
1315                 'upload_date': '20100430',
1316                 'uploader_id': 'deadmau5',
1317                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1318                 'creator': 'deadmau5',
1319                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1320                 'uploader': 'deadmau5',
1321                 'title': 'Deadmau5 - Some Chords (HD)',
1322                 'alt_title': 'Some Chords',
1323                 'availability': 'public',
1324                 'tags': 'count:14',
1325                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1326                 'view_count': int,
1327                 'live_status': 'not_live',
1328                 'channel': 'deadmau5',
1329                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1330                 'like_count': int,
1331                 'track': 'Some Chords',
1332                 'artist': 'deadmau5',
1333                 'playable_in_embed': True,
1334                 'age_limit': 0,
1335                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1336                 'categories': ['Music'],
1337                 'album': 'Some Chords',
1338                 'channel_follower_count': int
1339             },
1340             'expected_warnings': [
1341                 'DASH manifest missing',
1342             ]
1343         },
1344         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1345         {
1346             'url': 'lqQg6PlCWgI',
1347             'info_dict': {
1348                 'id': 'lqQg6PlCWgI',
1349                 'ext': 'mp4',
1350                 'duration': 6085,
1351                 'upload_date': '20150827',
1352                 'uploader_id': 'olympic',
1353                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1354                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1355                 'uploader': 'Olympics',
1356                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1357                 'like_count': int,
1358                 'release_timestamp': 1343767800,
1359                 'playable_in_embed': True,
1360                 'categories': ['Sports'],
1361                 'release_date': '20120731',
1362                 'channel': 'Olympics',
1363                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1364                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1365                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1366                 'age_limit': 0,
1367                 'availability': 'public',
1368                 'live_status': 'was_live',
1369                 'view_count': int,
1370                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1371                 'channel_follower_count': int
1372             },
1373             'params': {
1374                 'skip_download': 'requires avconv',
1375             }
1376         },
1377         # Non-square pixels
1378         {
1379             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1380             'info_dict': {
1381                 'id': '_b-2C3KPAM0',
1382                 'ext': 'mp4',
1383                 'stretched_ratio': 16 / 9.,
1384                 'duration': 85,
1385                 'upload_date': '20110310',
1386                 'uploader_id': 'AllenMeow',
1387                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1388                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1389                 'uploader': '孫ᄋᄅ',
1390                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1391                 'playable_in_embed': True,
1392                 'channel': '孫ᄋᄅ',
1393                 'age_limit': 0,
1394                 'tags': 'count:11',
1395                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1396                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1397                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1398                 'view_count': int,
1399                 'categories': ['People & Blogs'],
1400                 'like_count': int,
1401                 'live_status': 'not_live',
1402                 'availability': 'unlisted',
1403                 'comment_count': int,
1404                 'channel_follower_count': int
1405             },
1406         },
1407         # url_encoded_fmt_stream_map is empty string
1408         {
1409             'url': 'qEJwOuvDf7I',
1410             'info_dict': {
1411                 'id': 'qEJwOuvDf7I',
1412                 'ext': 'webm',
1413                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1414                 'description': '',
1415                 'upload_date': '20150404',
1416                 'uploader_id': 'spbelect',
1417                 'uploader': 'Наблюдатели Петербурга',
1418             },
1419             'params': {
1420                 'skip_download': 'requires avconv',
1421             },
1422             'skip': 'This live event has ended.',
1423         },
1424         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1425         {
1426             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1427             'info_dict': {
1428                 'id': 'FIl7x6_3R5Y',
1429                 'ext': 'webm',
1430                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1431                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1432                 'duration': 220,
1433                 'upload_date': '20150625',
1434                 'uploader_id': 'dorappi2000',
1435                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1436                 'uploader': 'dorappi2000',
1437                 'formats': 'mincount:31',
1438             },
1439             'skip': 'not actual anymore',
1440         },
1441         # DASH manifest with segment_list
1442         {
1443             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1444             'md5': '8ce563a1d667b599d21064e982ab9e31',
1445             'info_dict': {
1446                 'id': 'CsmdDsKjzN8',
1447                 'ext': 'mp4',
1448                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1449                 'uploader': 'Airtek',
1450                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1451                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1452                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1453             },
1454             'params': {
1455                 'youtube_include_dash_manifest': True,
1456                 'format': '135',  # bestvideo
1457             },
1458             'skip': 'This live event has ended.',
1459         },
1460         {
1461             # Multifeed videos (multiple cameras), URL is for Main Camera
1462             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1463             'info_dict': {
1464                 'id': 'jvGDaLqkpTg',
1465                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1466                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1467             },
1468             'playlist': [{
1469                 'info_dict': {
1470                     'id': 'jvGDaLqkpTg',
1471                     'ext': 'mp4',
1472                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1473                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1474                     'duration': 10643,
1475                     'upload_date': '20161111',
1476                     'uploader': 'Team PGP',
1477                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1478                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1479                 },
1480             }, {
1481                 'info_dict': {
1482                     'id': '3AKt1R1aDnw',
1483                     'ext': 'mp4',
1484                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1485                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1486                     'duration': 10991,
1487                     'upload_date': '20161111',
1488                     'uploader': 'Team PGP',
1489                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1490                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1491                 },
1492             }, {
1493                 'info_dict': {
1494                     'id': 'RtAMM00gpVc',
1495                     'ext': 'mp4',
1496                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1497                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1498                     'duration': 10995,
1499                     'upload_date': '20161111',
1500                     'uploader': 'Team PGP',
1501                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1502                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1503                 },
1504             }, {
1505                 'info_dict': {
1506                     'id': '6N2fdlP3C5U',
1507                     'ext': 'mp4',
1508                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1509                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1510                     'duration': 10990,
1511                     'upload_date': '20161111',
1512                     'uploader': 'Team PGP',
1513                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1514                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1515                 },
1516             }],
1517             'params': {
1518                 'skip_download': True,
1519             },
1520             'skip': 'Not multifeed anymore',
1521         },
1522         {
1523             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1524             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1525             'info_dict': {
1526                 'id': 'gVfLd0zydlo',
1527                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1528             },
1529             'playlist_count': 2,
1530             'skip': 'Not multifeed anymore',
1531         },
1532         {
1533             'url': 'https://vid.plus/FlRa-iH7PGw',
1534             'only_matching': True,
1535         },
1536         {
1537             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1538             'only_matching': True,
1539         },
1540         {
1541             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1542             # Also tests cut-off URL expansion in video description (see
1543             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1544             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1545             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1546             'info_dict': {
1547                 'id': 'lsguqyKfVQg',
1548                 'ext': 'mp4',
1549                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1550                 'alt_title': 'Dark Walk',
1551                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1552                 'duration': 133,
1553                 'upload_date': '20151119',
1554                 'uploader_id': 'IronSoulElf',
1555                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1556                 'uploader': 'IronSoulElf',
1557                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1558                 'track': 'Dark Walk',
1559                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1560                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1561                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1562                 'categories': ['Film & Animation'],
1563                 'view_count': int,
1564                 'live_status': 'not_live',
1565                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1566                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1567                 'tags': 'count:13',
1568                 'availability': 'public',
1569                 'channel': 'IronSoulElf',
1570                 'playable_in_embed': True,
1571                 'like_count': int,
1572                 'age_limit': 0,
1573                 'channel_follower_count': int
1574             },
1575             'params': {
1576                 'skip_download': True,
1577             },
1578         },
1579         {
1580             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1581             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1582             'only_matching': True,
1583         },
1584         {
1585             # Video with yt:stretch=17:0
1586             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1587             'info_dict': {
1588                 'id': 'Q39EVAstoRM',
1589                 'ext': 'mp4',
1590                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1591                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1592                 'upload_date': '20151107',
1593                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1594                 'uploader': 'CH GAMER DROID',
1595             },
1596             'params': {
1597                 'skip_download': True,
1598             },
1599             'skip': 'This video does not exist.',
1600         },
1601         {
1602             # Video with incomplete 'yt:stretch=16:'
1603             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1604             'only_matching': True,
1605         },
1606         {
1607             # Video licensed under Creative Commons
1608             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1609             'info_dict': {
1610                 'id': 'M4gD1WSo5mA',
1611                 'ext': 'mp4',
1612                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1613                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1614                 'duration': 721,
1615                 'upload_date': '20150128',
1616                 'uploader_id': 'BerkmanCenter',
1617                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1618                 'uploader': 'The Berkman Klein Center for Internet & Society',
1619                 'license': 'Creative Commons Attribution license (reuse allowed)',
1620                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1621                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1622                 'like_count': int,
1623                 'age_limit': 0,
1624                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1625                 'channel': 'The Berkman Klein Center for Internet & Society',
1626                 'availability': 'public',
1627                 'view_count': int,
1628                 'categories': ['Education'],
1629                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1630                 'live_status': 'not_live',
1631                 'playable_in_embed': True,
1632                 'comment_count': int,
1633                 'channel_follower_count': int
1634             },
1635             'params': {
1636                 'skip_download': True,
1637             },
1638         },
1639         {
1640             # Channel-like uploader_url
1641             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1642             'info_dict': {
1643                 'id': 'eQcmzGIKrzg',
1644                 'ext': 'mp4',
1645                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1646                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1647                 'duration': 4060,
1648                 'upload_date': '20151120',
1649                 'uploader': 'Bernie Sanders',
1650                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1651                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1652                 'license': 'Creative Commons Attribution license (reuse allowed)',
1653                 'playable_in_embed': True,
1654                 'tags': 'count:12',
1655                 'like_count': int,
1656                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1657                 'age_limit': 0,
1658                 'availability': 'public',
1659                 'categories': ['News & Politics'],
1660                 'channel': 'Bernie Sanders',
1661                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1662                 'view_count': int,
1663                 'live_status': 'not_live',
1664                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1665                 'comment_count': int,
1666                 'channel_follower_count': int
1667             },
1668             'params': {
1669                 'skip_download': True,
1670             },
1671         },
1672         {
1673             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1674             'only_matching': True,
1675         },
1676         {
1677             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1678             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1679             'only_matching': True,
1680         },
1681         {
1682             # Rental video preview
1683             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1684             'info_dict': {
1685                 'id': 'uGpuVWrhIzE',
1686                 'ext': 'mp4',
1687                 'title': 'Piku - Trailer',
1688                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1689                 'upload_date': '20150811',
1690                 'uploader': 'FlixMatrix',
1691                 'uploader_id': 'FlixMatrixKaravan',
1692                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1693                 'license': 'Standard YouTube License',
1694             },
1695             'params': {
1696                 'skip_download': True,
1697             },
1698             'skip': 'This video is not available.',
1699         },
1700         {
1701             # YouTube Red video with episode data
1702             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1703             'info_dict': {
1704                 'id': 'iqKdEhx-dD4',
1705                 'ext': 'mp4',
1706                 'title': 'Isolation - Mind Field (Ep 1)',
1707                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1708                 'duration': 2085,
1709                 'upload_date': '20170118',
1710                 'uploader': 'Vsauce',
1711                 'uploader_id': 'Vsauce',
1712                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1713                 'series': 'Mind Field',
1714                 'season_number': 1,
1715                 'episode_number': 1,
1716                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1717                 'tags': 'count:12',
1718                 'view_count': int,
1719                 'availability': 'public',
1720                 'age_limit': 0,
1721                 'channel': 'Vsauce',
1722                 'episode': 'Episode 1',
1723                 'categories': ['Entertainment'],
1724                 'season': 'Season 1',
1725                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1726                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1727                 'like_count': int,
1728                 'playable_in_embed': True,
1729                 'live_status': 'not_live',
1730                 'channel_follower_count': int
1731             },
1732             'params': {
1733                 'skip_download': True,
1734             },
1735             'expected_warnings': [
1736                 'Skipping DASH manifest',
1737             ],
1738         },
1739         {
1740             # The following content has been identified by the YouTube community
1741             # as inappropriate or offensive to some audiences.
1742             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1743             'info_dict': {
1744                 'id': '6SJNVb0GnPI',
1745                 'ext': 'mp4',
1746                 'title': 'Race Differences in Intelligence',
1747                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1748                 'duration': 965,
1749                 'upload_date': '20140124',
1750                 'uploader': 'New Century Foundation',
1751                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1752                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1753             },
1754             'params': {
1755                 'skip_download': True,
1756             },
1757             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1758         },
1759         {
1760             # itag 212
1761             'url': '1t24XAntNCY',
1762             'only_matching': True,
1763         },
1764         {
1765             # geo restricted to JP
1766             'url': 'sJL6WA-aGkQ',
1767             'only_matching': True,
1768         },
1769         {
1770             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1771             'only_matching': True,
1772         },
1773         {
1774             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1775             'only_matching': True,
1776         },
1777         {
1778             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1779             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1780             'only_matching': True,
1781         },
1782         {
1783             # DRM protected
1784             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1785             'only_matching': True,
1786         },
1787         {
1788             # Video with unsupported adaptive stream type formats
1789             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1790             'info_dict': {
1791                 'id': 'Z4Vy8R84T1U',
1792                 'ext': 'mp4',
1793                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1794                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1795                 'duration': 433,
1796                 'upload_date': '20130923',
1797                 'uploader': 'Amelia Putri Harwita',
1798                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1799                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1800                 'formats': 'maxcount:10',
1801             },
1802             'params': {
1803                 'skip_download': True,
1804                 'youtube_include_dash_manifest': False,
1805             },
1806             'skip': 'not actual anymore',
1807         },
1808         {
1809             # Youtube Music Auto-generated description
1810             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1811             'info_dict': {
1812                 'id': 'MgNrAu2pzNs',
1813                 'ext': 'mp4',
1814                 'title': 'Voyeur Girl',
1815                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1816                 'upload_date': '20190312',
1817                 'uploader': 'Stephen - Topic',
1818                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1819                 'artist': 'Stephen',
1820                 'track': 'Voyeur Girl',
1821                 'album': 'it\'s too much love to know my dear',
1822                 'release_date': '20190313',
1823                 'release_year': 2019,
1824                 'alt_title': 'Voyeur Girl',
1825                 'view_count': int,
1826                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1827                 'playable_in_embed': True,
1828                 'like_count': int,
1829                 'categories': ['Music'],
1830                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1831                 'channel': 'Stephen',
1832                 'availability': 'public',
1833                 'creator': 'Stephen',
1834                 'duration': 169,
1835                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1836                 'age_limit': 0,
1837                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1838                 'tags': 'count:11',
1839                 'live_status': 'not_live',
1840                 'channel_follower_count': int
1841             },
1842             'params': {
1843                 'skip_download': True,
1844             },
1845         },
1846         {
1847             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1848             'only_matching': True,
1849         },
1850         {
1851             # invalid -> valid video id redirection
1852             'url': 'DJztXj2GPfl',
1853             'info_dict': {
1854                 'id': 'DJztXj2GPfk',
1855                 'ext': 'mp4',
1856                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1857                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1858                 'upload_date': '20090125',
1859                 'uploader': 'Prochorowka',
1860                 'uploader_id': 'Prochorowka',
1861                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1862                 'artist': 'Panjabi MC',
1863                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1864                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1865             },
1866             'params': {
1867                 'skip_download': True,
1868             },
1869             'skip': 'Video unavailable',
1870         },
1871         {
1872             # empty description results in an empty string
1873             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1874             'info_dict': {
1875                 'id': 'x41yOUIvK2k',
1876                 'ext': 'mp4',
1877                 'title': 'IMG 3456',
1878                 'description': '',
1879                 'upload_date': '20170613',
1880                 'uploader_id': 'ElevageOrVert',
1881                 'uploader': 'ElevageOrVert',
1882                 'view_count': int,
1883                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1884                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1885                 'like_count': int,
1886                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1887                 'tags': [],
1888                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1889                 'availability': 'public',
1890                 'age_limit': 0,
1891                 'categories': ['Pets & Animals'],
1892                 'duration': 7,
1893                 'playable_in_embed': True,
1894                 'live_status': 'not_live',
1895                 'channel': 'ElevageOrVert',
1896                 'channel_follower_count': int
1897             },
1898             'params': {
1899                 'skip_download': True,
1900             },
1901         },
1902         {
1903             # with '};' inside yt initial data (see [1])
1904             # see [2] for an example with '};' inside ytInitialPlayerResponse
1905             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1906             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1907             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1908             'info_dict': {
1909                 'id': 'CHqg6qOn4no',
1910                 'ext': 'mp4',
1911                 'title': 'Part 77   Sort a list of simple types in c#',
1912                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1913                 'upload_date': '20130831',
1914                 'uploader_id': 'kudvenkat',
1915                 'uploader': 'kudvenkat',
1916                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1917                 'like_count': int,
1918                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1919                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1920                 'live_status': 'not_live',
1921                 'categories': ['Education'],
1922                 'availability': 'public',
1923                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1924                 'tags': 'count:12',
1925                 'playable_in_embed': True,
1926                 'age_limit': 0,
1927                 'view_count': int,
1928                 'duration': 522,
1929                 'channel': 'kudvenkat',
1930                 'comment_count': int,
1931                 'channel_follower_count': int
1932             },
1933             'params': {
1934                 'skip_download': True,
1935             },
1936         },
1937         {
1938             # another example of '};' in ytInitialData
1939             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1940             'only_matching': True,
1941         },
1942         {
1943             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1944             'only_matching': True,
1945         },
1946         {
1947             # https://github.com/ytdl-org/youtube-dl/pull/28094
1948             'url': 'OtqTfy26tG0',
1949             'info_dict': {
1950                 'id': 'OtqTfy26tG0',
1951                 'ext': 'mp4',
1952                 'title': 'Burn Out',
1953                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1954                 'upload_date': '20141120',
1955                 'uploader': 'The Cinematic Orchestra - Topic',
1956                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1957                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1958                 'artist': 'The Cinematic Orchestra',
1959                 'track': 'Burn Out',
1960                 'album': 'Every Day',
1961                 'like_count': int,
1962                 'live_status': 'not_live',
1963                 'alt_title': 'Burn Out',
1964                 'duration': 614,
1965                 'age_limit': 0,
1966                 'view_count': int,
1967                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1968                 'creator': 'The Cinematic Orchestra',
1969                 'channel': 'The Cinematic Orchestra',
1970                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1971                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1972                 'availability': 'public',
1973                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1974                 'categories': ['Music'],
1975                 'playable_in_embed': True,
1976                 'channel_follower_count': int
1977             },
1978             'params': {
1979                 'skip_download': True,
1980             },
1981         },
1982         {
1983             # controversial video, only works with bpctr when authenticated with cookies
1984             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1985             'only_matching': True,
1986         },
1987         {
1988             # controversial video, requires bpctr/contentCheckOk
1989             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1990             'info_dict': {
1991                 'id': 'SZJvDhaSDnc',
1992                 'ext': 'mp4',
1993                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1994                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1995                 'uploader': 'CBS Mornings',
1996                 'uploader_id': 'CBSThisMorning',
1997                 'upload_date': '20140716',
1998                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1999                 'duration': 170,
2000                 'categories': ['News & Politics'],
2001                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2002                 'view_count': int,
2003                 'channel': 'CBS Mornings',
2004                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2005                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2006                 'age_limit': 18,
2007                 'availability': 'needs_auth',
2008                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2009                 'like_count': int,
2010                 'live_status': 'not_live',
2011                 'playable_in_embed': True,
2012                 'channel_follower_count': int
2013             }
2014         },
2015         {
2016             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2017             'url': 'cBvYw8_A0vQ',
2018             'info_dict': {
2019                 'id': 'cBvYw8_A0vQ',
2020                 'ext': 'mp4',
2021                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2022                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2023                 'upload_date': '20201120',
2024                 'uploader': 'Walk around Japan',
2025                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2026                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2027                 'duration': 1456,
2028                 'categories': ['Travel & Events'],
2029                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2030                 'view_count': int,
2031                 'channel': 'Walk around Japan',
2032                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2033                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2034                 'age_limit': 0,
2035                 'availability': 'public',
2036                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2037                 'live_status': 'not_live',
2038                 'playable_in_embed': True,
2039                 'channel_follower_count': int
2040             },
2041             'params': {
2042                 'skip_download': True,
2043             },
2044         }, {
2045             # Has multiple audio streams
2046             'url': 'WaOKSUlf4TM',
2047             'only_matching': True
2048         }, {
2049             # Requires Premium: has format 141 when requested using YTM url
2050             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2051             'only_matching': True
2052         }, {
2053             # multiple subtitles with same lang_code
2054             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2055             'only_matching': True,
2056         }, {
2057             # Force use android client fallback
2058             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2059             'info_dict': {
2060                 'id': 'YOelRv7fMxY',
2061                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2062                 'ext': '3gp',
2063                 'upload_date': '20210624',
2064                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2065                 'uploader': 'colinfurze',
2066                 'uploader_id': 'colinfurze',
2067                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2068                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2069                 'duration': 596,
2070                 'categories': ['Entertainment'],
2071                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2072                 'view_count': int,
2073                 'channel': 'colinfurze',
2074                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2075                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2076                 'age_limit': 0,
2077                 'availability': 'public',
2078                 'like_count': int,
2079                 'live_status': 'not_live',
2080                 'playable_in_embed': True,
2081                 'channel_follower_count': int
2082             },
2083             'params': {
2084                 'format': '17',  # 3gp format available on android
2085                 'extractor_args': {'youtube': {'player_client': ['android']}},
2086             },
2087         },
2088         {
2089             # Skip download of additional client configs (remix client config in this case)
2090             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2091             'only_matching': True,
2092             'params': {
2093                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2094             },
2095         }, {
2096             # shorts
2097             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2098             'only_matching': True,
2099         }, {
2100             'note': 'Storyboards',
2101             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2102             'info_dict': {
2103                 'id': '5KLPxDtMqe8',
2104                 'ext': 'mhtml',
2105                 'format_id': 'sb0',
2106                 'title': 'Your Brain is Plastic',
2107                 'uploader_id': 'scishow',
2108                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2109                 'upload_date': '20140324',
2110                 'uploader': 'SciShow',
2111                 'like_count': int,
2112                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2113                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2114                 'view_count': int,
2115                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2116                 'playable_in_embed': True,
2117                 'tags': 'count:12',
2118                 'uploader_url': 'http://www.youtube.com/user/scishow',
2119                 'availability': 'public',
2120                 'channel': 'SciShow',
2121                 'live_status': 'not_live',
2122                 'duration': 248,
2123                 'categories': ['Education'],
2124                 'age_limit': 0,
2125                 'channel_follower_count': int
2126             }, 'params': {'format': 'mhtml', 'skip_download': True}
2127         }, {
2128             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2129             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2130             'info_dict': {
2131                 'id': '2NUZ8W2llS4',
2132                 'ext': 'mp4',
2133                 'title': 'The NP that test your phone performance 🙂',
2134                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2135                 'uploader': 'Leon Nguyen',
2136                 'uploader_id': 'VNSXIII',
2137                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2138                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2139                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2140                 'duration': 21,
2141                 'view_count': int,
2142                 'age_limit': 0,
2143                 'categories': ['Gaming'],
2144                 'tags': 'count:23',
2145                 'playable_in_embed': True,
2146                 'live_status': 'not_live',
2147                 'upload_date': '20220103',
2148                 'like_count': int,
2149                 'availability': 'public',
2150                 'channel': 'Leon Nguyen',
2151                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2152                 'comment_count': int,
2153                 'channel_follower_count': int
2154             }
2155         }, {
2156             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2157             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2158             'info_dict': {
2159                 'id': 'mzZzzBU6lrM',
2160                 'ext': 'mp4',
2161                 'title': 'I Met GeorgeNotFound In Real Life...',
2162                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2163                 'uploader': 'Quackity',
2164                 'uploader_id': 'QuackityHQ',
2165                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2166                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2167                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2168                 'duration': 955,
2169                 'view_count': int,
2170                 'age_limit': 0,
2171                 'categories': ['Entertainment'],
2172                 'tags': 'count:26',
2173                 'playable_in_embed': True,
2174                 'live_status': 'not_live',
2175                 'release_timestamp': 1641172509,
2176                 'release_date': '20220103',
2177                 'upload_date': '20220103',
2178                 'like_count': int,
2179                 'availability': 'public',
2180                 'channel': 'Quackity',
2181                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2182                 'channel_follower_count': int
2183             }
2184         },
2185         {   # continuous livestream. Microformat upload date should be preferred.
2186             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2187             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2188             'info_dict': {
2189                 'id': 'kgx4WGK0oNU',
2190                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2191                 'ext': 'mp4',
2192                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2193                 'availability': 'public',
2194                 'age_limit': 0,
2195                 'release_timestamp': 1637975704,
2196                 'upload_date': '20210619',
2197                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2198                 'live_status': 'is_live',
2199                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2200                 'uploader': '阿鲍Abao',
2201                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2202                 'channel': 'Abao in Tokyo',
2203                 'channel_follower_count': int,
2204                 'release_date': '20211127',
2205                 'tags': 'count:39',
2206                 'categories': ['People & Blogs'],
2207                 'like_count': int,
2208                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2209                 'view_count': int,
2210                 'playable_in_embed': True,
2211                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2212             },
2213             'params': {'skip_download': True}
2214         }, {
2215             # Story. Requires specific player params to work.
2216             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2217             'info_dict': {
2218                 'id': 'vv8qTUWmulI',
2219                 'ext': 'mp4',
2220                 'availability': 'unlisted',
2221                 'view_count': int,
2222                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2223                 'upload_date': '20220526',
2224                 'categories': ['Education'],
2225                 'title': 'Story',
2226                 'channel': 'IT\'S HISTORY',
2227                 'description': '',
2228                 'uploader_id': 'BlastfromthePast',
2229                 'duration': 12,
2230                 'uploader': 'IT\'S HISTORY',
2231                 'playable_in_embed': True,
2232                 'age_limit': 0,
2233                 'live_status': 'not_live',
2234                 'tags': [],
2235                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2236                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2237                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2238             },
2239             'skip': 'stories get removed after some period of time',
2240         }, {
2241             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2242             'info_dict': {
2243                 'id': 'tjjjtzRLHvA',
2244                 'ext': 'mp4',
2245                 'title': 'ハッシュタグ無し };if window.ytcsi',
2246                 'upload_date': '20220323',
2247                 'like_count': int,
2248                 'availability': 'unlisted',
2249                 'channel': 'nao20010128nao',
2250                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2251                 'age_limit': 0,
2252                 'uploader': 'nao20010128nao',
2253                 'uploader_id': 'nao20010128nao',
2254                 'categories': ['Music'],
2255                 'view_count': int,
2256                 'description': '',
2257                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2258                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2259                 'live_status': 'not_live',
2260                 'playable_in_embed': True,
2261                 'channel_follower_count': int,
2262                 'duration': 6,
2263                 'tags': [],
2264                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2265             }
2266         }
2267     ]
2268
2269     @classmethod
2270     def suitable(cls, url):
2271         from ..utils import parse_qs
2272
2273         qs = parse_qs(url)
2274         if qs.get('list', [None])[0]:
2275             return False
2276         return super().suitable(url)
2277
2278     def __init__(self, *args, **kwargs):
2279         super().__init__(*args, **kwargs)
2280         self._code_cache = {}
2281         self._player_cache = {}
2282
2283     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2284         lock = threading.Lock()
2285
2286         is_live = True
2287         start_time = time.time()
2288         formats = [f for f in formats if f.get('is_from_start')]
2289
2290         def refetch_manifest(format_id, delay):
2291             nonlocal formats, start_time, is_live
2292             if time.time() <= start_time + delay:
2293                 return
2294
2295             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2296             video_details = traverse_obj(
2297                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2298             microformats = traverse_obj(
2299                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2300                 expected_type=dict, default=[])
2301             _, is_live, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2302             start_time = time.time()
2303
2304         def mpd_feed(format_id, delay):
2305             """
2306             @returns (manifest_url, manifest_stream_number, is_live) or None
2307             """
2308             with lock:
2309                 refetch_manifest(format_id, delay)
2310
2311             f = next((f for f in formats if f['format_id'] == format_id), None)
2312             if not f:
2313                 if not is_live:
2314                     self.to_screen(f'{video_id}: Video is no longer live')
2315                 else:
2316                     self.report_warning(
2317                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2318                 return None
2319             return f['manifest_url'], f['manifest_stream_number'], is_live
2320
2321         for f in formats:
2322             f['is_live'] = True
2323             f['protocol'] = 'http_dash_segments_generator'
2324             f['fragments'] = functools.partial(
2325                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2326
2327     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2328         FETCH_SPAN, MAX_DURATION = 5, 432000
2329
2330         mpd_url, stream_number, is_live = None, None, True
2331
2332         begin_index = 0
2333         download_start_time = ctx.get('start') or time.time()
2334
2335         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2336         if lack_early_segments:
2337             self.report_warning(bug_reports_message(
2338                 'Starting download from the last 120 hours of the live stream since '
2339                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2340             lack_early_segments = True
2341
2342         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2343         fragments, fragment_base_url = None, None
2344
2345         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2346             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2347             # Obtain from MPD's maximum seq value
2348             old_mpd_url = mpd_url
2349             last_error = ctx.pop('last_error', None)
2350             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2351             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2352                                                or (mpd_url, stream_number, False))
2353             if not refresh_sequence:
2354                 if expire_fast and not is_live:
2355                     return False, last_seq
2356                 elif old_mpd_url == mpd_url:
2357                     return True, last_seq
2358             try:
2359                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2360                     mpd_url, None, note=False, errnote=False, fatal=False)
2361             except ExtractorError:
2362                 fmts = None
2363             if not fmts:
2364                 no_fragment_score += 2
2365                 return False, last_seq
2366             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2367             fragments = fmt_info['fragments']
2368             fragment_base_url = fmt_info['fragment_base_url']
2369             assert fragment_base_url
2370
2371             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2372             return True, _last_seq
2373
2374         while is_live:
2375             fetch_time = time.time()
2376             if no_fragment_score > 30:
2377                 return
2378             if last_segment_url:
2379                 # Obtain from "X-Head-Seqnum" header value from each segment
2380                 try:
2381                     urlh = self._request_webpage(
2382                         last_segment_url, None, note=False, errnote=False, fatal=False)
2383                 except ExtractorError:
2384                     urlh = None
2385                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2386                 if last_seq is None:
2387                     no_fragment_score += 2
2388                     last_segment_url = None
2389                     continue
2390             else:
2391                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2392                 no_fragment_score += 2
2393                 if not should_continue:
2394                     continue
2395
2396             if known_idx > last_seq:
2397                 last_segment_url = None
2398                 continue
2399
2400             last_seq += 1
2401
2402             if begin_index < 0 and known_idx < 0:
2403                 # skip from the start when it's negative value
2404                 known_idx = last_seq + begin_index
2405             if lack_early_segments:
2406                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2407             try:
2408                 for idx in range(known_idx, last_seq):
2409                     # do not update sequence here or you'll get skipped some part of it
2410                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2411                     if not should_continue:
2412                         known_idx = idx - 1
2413                         raise ExtractorError('breaking out of outer loop')
2414                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2415                     yield {
2416                         'url': last_segment_url,
2417                         'fragment_count': last_seq,
2418                     }
2419                 if known_idx == last_seq:
2420                     no_fragment_score += 5
2421                 else:
2422                     no_fragment_score = 0
2423                 known_idx = last_seq
2424             except ExtractorError:
2425                 continue
2426
2427             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2428
2429     def _extract_player_url(self, *ytcfgs, webpage=None):
2430         player_url = traverse_obj(
2431             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2432             get_all=False, expected_type=str)
2433         if not player_url:
2434             return
2435         return urljoin('https://www.youtube.com', player_url)
2436
2437     def _download_player_url(self, video_id, fatal=False):
2438         res = self._download_webpage(
2439             'https://www.youtube.com/iframe_api',
2440             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2441         if res:
2442             player_version = self._search_regex(
2443                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2444             if player_version:
2445                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2446
2447     def _signature_cache_id(self, example_sig):
2448         """ Return a string representation of a signature """
2449         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2450
2451     @classmethod
2452     def _extract_player_info(cls, player_url):
2453         for player_re in cls._PLAYER_INFO_RE:
2454             id_m = re.search(player_re, player_url)
2455             if id_m:
2456                 break
2457         else:
2458             raise ExtractorError('Cannot identify player %r' % player_url)
2459         return id_m.group('id')
2460
2461     def _load_player(self, video_id, player_url, fatal=True):
2462         player_id = self._extract_player_info(player_url)
2463         if player_id not in self._code_cache:
2464             code = self._download_webpage(
2465                 player_url, video_id, fatal=fatal,
2466                 note='Downloading player ' + player_id,
2467                 errnote='Download of %s failed' % player_url)
2468             if code:
2469                 self._code_cache[player_id] = code
2470         return self._code_cache.get(player_id)
2471
2472     def _extract_signature_function(self, video_id, player_url, example_sig):
2473         player_id = self._extract_player_info(player_url)
2474
2475         # Read from filesystem cache
2476         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2477         assert os.path.basename(func_id) == func_id
2478
2479         self.write_debug(f'Extracting signature function {func_id}')
2480         cache_spec = self.cache.load('youtube-sigfuncs', func_id)
2481         if cache_spec is not None:
2482             return lambda s: ''.join(s[i] for i in cache_spec)
2483
2484         code = self._load_player(video_id, player_url)
2485         if code:
2486             res = self._parse_sig_js(code)
2487
2488             test_string = ''.join(map(chr, range(len(example_sig))))
2489             cache_res = res(test_string)
2490             cache_spec = [ord(c) for c in cache_res]
2491
2492             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2493             return res
2494
2495     def _print_sig_code(self, func, example_sig):
2496         if not self.get_param('youtube_print_sig_code'):
2497             return
2498
2499         def gen_sig_code(idxs):
2500             def _genslice(start, end, step):
2501                 starts = '' if start == 0 else str(start)
2502                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2503                 steps = '' if step == 1 else (':%d' % step)
2504                 return f's[{starts}{ends}{steps}]'
2505
2506             step = None
2507             # Quelch pyflakes warnings - start will be set when step is set
2508             start = '(Never used)'
2509             for i, prev in zip(idxs[1:], idxs[:-1]):
2510                 if step is not None:
2511                     if i - prev == step:
2512                         continue
2513                     yield _genslice(start, prev, step)
2514                     step = None
2515                     continue
2516                 if i - prev in [-1, 1]:
2517                     step = i - prev
2518                     start = prev
2519                     continue
2520                 else:
2521                     yield 's[%d]' % prev
2522             if step is None:
2523                 yield 's[%d]' % i
2524             else:
2525                 yield _genslice(start, i, step)
2526
2527         test_string = ''.join(map(chr, range(len(example_sig))))
2528         cache_res = func(test_string)
2529         cache_spec = [ord(c) for c in cache_res]
2530         expr_code = ' + '.join(gen_sig_code(cache_spec))
2531         signature_id_tuple = '(%s)' % (
2532             ', '.join(str(len(p)) for p in example_sig.split('.')))
2533         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2534                 '    return %s\n') % (signature_id_tuple, expr_code)
2535         self.to_screen('Extracted signature function:\n' + code)
2536
2537     def _parse_sig_js(self, jscode):
2538         funcname = self._search_regex(
2539             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2540              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2541              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2542              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2543              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2544              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2545              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2546              # Obsolete patterns
2547              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2548              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2549              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2550              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2551              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2552              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2553              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2554              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2555             jscode, 'Initial JS player signature function name', group='sig')
2556
2557         jsi = JSInterpreter(jscode)
2558         initial_function = jsi.extract_function(funcname)
2559         return lambda s: initial_function([s])
2560
2561     def _decrypt_signature(self, s, video_id, player_url):
2562         """Turn the encrypted s field into a working signature"""
2563         try:
2564             player_id = (player_url, self._signature_cache_id(s))
2565             if player_id not in self._player_cache:
2566                 func = self._extract_signature_function(video_id, player_url, s)
2567                 self._player_cache[player_id] = func
2568             func = self._player_cache[player_id]
2569             self._print_sig_code(func, s)
2570             return func(s)
2571         except Exception as e:
2572             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2573
2574     def _decrypt_nsig(self, s, video_id, player_url):
2575         """Turn the encrypted n field into a working signature"""
2576         if player_url is None:
2577             raise ExtractorError('Cannot decrypt nsig without player_url')
2578         player_url = urljoin('https://www.youtube.com', player_url)
2579
2580         sig_id = ('nsig_value', s)
2581         if sig_id in self._player_cache:
2582             return self._player_cache[sig_id]
2583
2584         try:
2585             player_id = ('nsig', player_url)
2586             if player_id not in self._player_cache:
2587                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2588             func = self._player_cache[player_id]
2589             self._player_cache[sig_id] = func(s)
2590             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2591             return self._player_cache[sig_id]
2592         except Exception as e:
2593             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2594
2595     def _extract_n_function_name(self, jscode):
2596         nfunc, idx = self._search_regex(
2597             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2598             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2599         if not idx:
2600             return nfunc
2601         return json.loads(js_to_json(self._search_regex(
2602             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2603             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2604
2605     def _extract_n_function(self, video_id, player_url):
2606         player_id = self._extract_player_info(player_url)
2607         func_code = self.cache.load('youtube-nsig', player_id)
2608
2609         if func_code:
2610             jsi = JSInterpreter(func_code)
2611         else:
2612             jscode = self._load_player(video_id, player_url)
2613             funcname = self._extract_n_function_name(jscode)
2614             jsi = JSInterpreter(jscode)
2615             func_code = jsi.extract_function_code(funcname)
2616             self.cache.store('youtube-nsig', player_id, func_code)
2617
2618         if self.get_param('youtube_print_sig_code'):
2619             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2620
2621         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2622
2623     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2624         """
2625         Extract signatureTimestamp (sts)
2626         Required to tell API what sig/player version is in use.
2627         """
2628         sts = None
2629         if isinstance(ytcfg, dict):
2630             sts = int_or_none(ytcfg.get('STS'))
2631
2632         if not sts:
2633             # Attempt to extract from player
2634             if player_url is None:
2635                 error_msg = 'Cannot extract signature timestamp without player_url.'
2636                 if fatal:
2637                     raise ExtractorError(error_msg)
2638                 self.report_warning(error_msg)
2639                 return
2640             code = self._load_player(video_id, player_url, fatal=fatal)
2641             if code:
2642                 sts = int_or_none(self._search_regex(
2643                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2644                     'JS player signature timestamp', group='sts', fatal=fatal))
2645         return sts
2646
2647     def _mark_watched(self, video_id, player_responses):
2648         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2649             label = 'fully ' if is_full else ''
2650             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2651                             expected_type=url_or_none)
2652             if not url:
2653                 self.report_warning(f'Unable to mark {label}watched')
2654                 return
2655             parsed_url = urllib.parse.urlparse(url)
2656             qs = urllib.parse.parse_qs(parsed_url.query)
2657
2658             # cpn generation algorithm is reverse engineered from base.js.
2659             # In fact it works even with dummy cpn.
2660             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2661             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2662
2663             # # more consistent results setting it to right before the end
2664             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2665
2666             qs.update({
2667                 'ver': ['2'],
2668                 'cpn': [cpn],
2669                 'cmt': video_length,
2670                 'el': 'detailpage',  # otherwise defaults to "shorts"
2671             })
2672
2673             if is_full:
2674                 # these seem to mark watchtime "history" in the real world
2675                 # they're required, so send in a single value
2676                 qs.update({
2677                     'st': video_length,
2678                     'et': video_length,
2679                 })
2680
2681             url = urllib.parse.urlunparse(
2682                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2683
2684             self._download_webpage(
2685                 url, video_id, f'Marking {label}watched',
2686                 'Unable to mark watched', fatal=False)
2687
2688     @staticmethod
2689     def _extract_urls(webpage):
2690         # Embedded YouTube player
2691         entries = [
2692             unescapeHTML(mobj.group('url'))
2693             for mobj in re.finditer(r'''(?x)
2694             (?:
2695                 <iframe[^>]+?src=|
2696                 data-video-url=|
2697                 <embed[^>]+?src=|
2698                 embedSWF\(?:\s*|
2699                 <object[^>]+data=|
2700                 new\s+SWFObject\(
2701             )
2702             (["\'])
2703                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2704                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2705             \1''', webpage)]
2706
2707         # lazyYT YouTube embed
2708         entries.extend(list(map(
2709             unescapeHTML,
2710             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2711
2712         # Wordpress "YouTube Video Importer" plugin
2713         matches = re.findall(r'''(?x)<div[^>]+
2714             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2715             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2716         entries.extend(m[-1] for m in matches)
2717
2718         return entries
2719
2720     @staticmethod
2721     def _extract_url(webpage):
2722         urls = YoutubeIE._extract_urls(webpage)
2723         return urls[0] if urls else None
2724
2725     @classmethod
2726     def extract_id(cls, url):
2727         video_id = cls.get_temp_id(url)
2728         if not video_id:
2729             raise ExtractorError(f'Invalid URL: {url}')
2730         return video_id
2731
2732     def _extract_chapters_from_json(self, data, duration):
2733         chapter_list = traverse_obj(
2734             data, (
2735                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2736                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2737             ), expected_type=list)
2738
2739         return self._extract_chapters(
2740             chapter_list,
2741             chapter_time=lambda chapter: float_or_none(
2742                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2743             chapter_title=lambda chapter: traverse_obj(
2744                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2745             duration=duration)
2746
2747     def _extract_chapters_from_engagement_panel(self, data, duration):
2748         content_list = traverse_obj(
2749             data,
2750             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2751             expected_type=list, default=[])
2752         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2753         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2754
2755         return next(filter(None, (
2756             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2757                                    chapter_time, chapter_title, duration)
2758             for contents in content_list)), [])
2759
2760     def _extract_chapters_from_description(self, description, duration):
2761         return self._extract_chapters(
2762             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
2763             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
2764             duration=duration, strict=False)
2765
2766     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
2767         if not duration:
2768             return
2769         chapter_list = [{
2770             'start_time': chapter_time(chapter),
2771             'title': chapter_title(chapter),
2772         } for chapter in chapter_list or []]
2773         if not strict:
2774             chapter_list.sort(key=lambda c: c['start_time'] or 0)
2775
2776         chapters = [{'start_time': 0}]
2777         for idx, chapter in enumerate(chapter_list):
2778             if chapter['start_time'] is None:
2779                 self.report_warning(f'Incomplete chapter {idx}')
2780             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
2781                 chapters.append(chapter)
2782             else:
2783                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
2784         return chapters[1:]
2785
2786     def _extract_comment(self, comment_renderer, parent=None):
2787         comment_id = comment_renderer.get('commentId')
2788         if not comment_id:
2789             return
2790
2791         text = self._get_text(comment_renderer, 'contentText')
2792
2793         # note: timestamp is an estimate calculated from the current time and time_text
2794         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2795         author = self._get_text(comment_renderer, 'authorText')
2796         author_id = try_get(comment_renderer,
2797                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
2798
2799         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2800                                                        lambda x: x['likeCount']), str)) or 0
2801         author_thumbnail = try_get(comment_renderer,
2802                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
2803
2804         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2805         is_favorited = 'creatorHeart' in (try_get(
2806             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2807         return {
2808             'id': comment_id,
2809             'text': text,
2810             'timestamp': timestamp,
2811             'time_text': time_text,
2812             'like_count': votes,
2813             'is_favorited': is_favorited,
2814             'author': author,
2815             'author_id': author_id,
2816             'author_thumbnail': author_thumbnail,
2817             'author_is_uploader': author_is_uploader,
2818             'parent': parent or 'root'
2819         }
2820
2821     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2822
2823         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2824
2825         def extract_header(contents):
2826             _continuation = None
2827             for content in contents:
2828                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2829                 expected_comment_count = self._get_count(
2830                     comments_header_renderer, 'countText', 'commentsCount')
2831
2832                 if expected_comment_count:
2833                     tracker['est_total'] = expected_comment_count
2834                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2835                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2836
2837                 sort_menu_item = try_get(
2838                     comments_header_renderer,
2839                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2840                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2841
2842                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2843                 if not _continuation:
2844                     continue
2845
2846                 sort_text = str_or_none(sort_menu_item.get('title'))
2847                 if not sort_text:
2848                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2849                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2850                 break
2851             return _continuation
2852
2853         def extract_thread(contents):
2854             if not parent:
2855                 tracker['current_page_thread'] = 0
2856             for content in contents:
2857                 if not parent and tracker['total_parent_comments'] >= max_parents:
2858                     yield
2859                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2860                 comment_renderer = get_first(
2861                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2862                     expected_type=dict, default={})
2863
2864                 comment = self._extract_comment(comment_renderer, parent)
2865                 if not comment:
2866                     continue
2867
2868                 tracker['running_total'] += 1
2869                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2870                 yield comment
2871
2872                 # Attempt to get the replies
2873                 comment_replies_renderer = try_get(
2874                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2875
2876                 if comment_replies_renderer:
2877                     tracker['current_page_thread'] += 1
2878                     comment_entries_iter = self._comment_entries(
2879                         comment_replies_renderer, ytcfg, video_id,
2880                         parent=comment.get('id'), tracker=tracker)
2881                     yield from itertools.islice(comment_entries_iter, min(
2882                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2883
2884         # Keeps track of counts across recursive calls
2885         if not tracker:
2886             tracker = dict(
2887                 running_total=0,
2888                 est_total=0,
2889                 current_page_thread=0,
2890                 total_parent_comments=0,
2891                 total_reply_comments=0)
2892
2893         # TODO: Deprecated
2894         # YouTube comments have a max depth of 2
2895         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2896         if max_depth:
2897             self._downloader.deprecation_warning(
2898                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2899         if max_depth == 1 and parent:
2900             return
2901
2902         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2903             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2904
2905         continuation = self._extract_continuation(root_continuation_data)
2906
2907         response = None
2908         is_forced_continuation = False
2909         is_first_continuation = parent is None
2910         if is_first_continuation and not continuation:
2911             # Sometimes you can get comments by generating the continuation yourself,
2912             # even if YouTube initially reports them being disabled - e.g. stories comments.
2913             # Note: if the comment section is actually disabled, YouTube may return a response with
2914             # required check_get_keys missing. So we will disable that check initially in this case.
2915             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2916             is_forced_continuation = True
2917
2918         for page_num in itertools.count(0):
2919             if not continuation:
2920                 break
2921             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2922             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2923             if page_num == 0:
2924                 if is_first_continuation:
2925                     note_prefix = 'Downloading comment section API JSON'
2926                 else:
2927                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2928                         tracker['current_page_thread'], comment_prog_str)
2929             else:
2930                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2931                     '       ' if parent else '', ' replies' if parent else '',
2932                     page_num, comment_prog_str)
2933
2934             response = self._extract_response(
2935                 item_id=None, query=continuation,
2936                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2937                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2938             is_forced_continuation = False
2939             continuation_contents = traverse_obj(
2940                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2941
2942             continuation = None
2943             for continuation_section in continuation_contents:
2944                 continuation_items = traverse_obj(
2945                     continuation_section,
2946                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2947                     get_all=False, expected_type=list) or []
2948                 if is_first_continuation:
2949                     continuation = extract_header(continuation_items)
2950                     is_first_continuation = False
2951                     if continuation:
2952                         break
2953                     continue
2954
2955                 for entry in extract_thread(continuation_items):
2956                     if not entry:
2957                         return
2958                     yield entry
2959                 continuation = self._extract_continuation({'contents': continuation_items})
2960                 if continuation:
2961                     break
2962
2963         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2964         if message and not parent and tracker['running_total'] == 0:
2965             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2966
2967     @staticmethod
2968     def _generate_comment_continuation(video_id):
2969         """
2970         Generates initial comment section continuation token from given video id
2971         """
2972         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2973         return base64.b64encode(token.encode()).decode()
2974
2975     def _get_comments(self, ytcfg, video_id, contents, webpage):
2976         """Entry for comment extraction"""
2977         def _real_comment_extract(contents):
2978             renderer = next((
2979                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2980                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2981             yield from self._comment_entries(renderer, ytcfg, video_id)
2982
2983         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2984         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2985
2986     @staticmethod
2987     def _get_checkok_params():
2988         return {'contentCheckOk': True, 'racyCheckOk': True}
2989
2990     @classmethod
2991     def _generate_player_context(cls, sts=None):
2992         context = {
2993             'html5Preference': 'HTML5_PREF_WANTS',
2994         }
2995         if sts is not None:
2996             context['signatureTimestamp'] = sts
2997         return {
2998             'playbackContext': {
2999                 'contentPlaybackContext': context
3000             },
3001             **cls._get_checkok_params()
3002         }
3003
3004     @staticmethod
3005     def _is_agegated(player_response):
3006         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3007             return True
3008
3009         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3010         AGE_GATE_REASONS = (
3011             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3012             'age_verification_required', 'age_check_required',  # status
3013         )
3014         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3015
3016     @staticmethod
3017     def _is_unplayable(player_response):
3018         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3019
3020     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
3021
3022         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3023         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3024         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3025         headers = self.generate_api_headers(
3026             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3027
3028         yt_query = {
3029             'videoId': video_id,
3030             'params': '8AEB'  # enable stories
3031         }
3032         yt_query.update(self._generate_player_context(sts))
3033         return self._extract_response(
3034             item_id=video_id, ep='player', query=yt_query,
3035             ytcfg=player_ytcfg, headers=headers, fatal=True,
3036             default_client=client,
3037             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3038         ) or None
3039
3040     def _get_requested_clients(self, url, smuggled_data):
3041         requested_clients = []
3042         default = ['android', 'web']
3043         allowed_clients = sorted(
3044             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3045             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3046         for client in self._configuration_arg('player_client'):
3047             if client in allowed_clients:
3048                 requested_clients.append(client)
3049             elif client == 'default':
3050                 requested_clients.extend(default)
3051             elif client == 'all':
3052                 requested_clients.extend(allowed_clients)
3053             else:
3054                 self.report_warning(f'Skipping unsupported client {client}')
3055         if not requested_clients:
3056             requested_clients = default
3057
3058         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3059             requested_clients.extend(
3060                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3061
3062         return orderedSet(requested_clients)
3063
3064     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3065         initial_pr = None
3066         if webpage:
3067             initial_pr = self._search_json(
3068                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3069
3070         all_clients = set(clients)
3071         clients = clients[::-1]
3072         prs = []
3073
3074         def append_client(*client_names):
3075             """ Append the first client name that exists but not already used """
3076             for client_name in client_names:
3077                 actual_client = _split_innertube_client(client_name)[0]
3078                 if actual_client in INNERTUBE_CLIENTS:
3079                     if actual_client not in all_clients:
3080                         clients.append(client_name)
3081                         all_clients.add(actual_client)
3082                         return
3083
3084         # Android player_response does not have microFormats which are needed for
3085         # extraction of some data. So we return the initial_pr with formats
3086         # stripped out even if not requested by the user
3087         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3088         if initial_pr:
3089             pr = dict(initial_pr)
3090             pr['streamingData'] = None
3091             prs.append(pr)
3092
3093         last_error = None
3094         tried_iframe_fallback = False
3095         player_url = None
3096         while clients:
3097             client, base_client, variant = _split_innertube_client(clients.pop())
3098             player_ytcfg = master_ytcfg if client == 'web' else {}
3099             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3100                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3101
3102             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3103             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3104             if 'js' in self._configuration_arg('player_skip'):
3105                 require_js_player = False
3106                 player_url = None
3107
3108             if not player_url and not tried_iframe_fallback and require_js_player:
3109                 player_url = self._download_player_url(video_id)
3110                 tried_iframe_fallback = True
3111
3112             try:
3113                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3114                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3115             except ExtractorError as e:
3116                 if last_error:
3117                     self.report_warning(last_error)
3118                 last_error = e
3119                 continue
3120
3121             if pr:
3122                 prs.append(pr)
3123
3124             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3125             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3126                 append_client(f'{base_client}_creator')
3127             elif self._is_agegated(pr):
3128                 if variant == 'tv_embedded':
3129                     append_client(f'{base_client}_embedded')
3130                 elif not variant:
3131                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3132
3133         if last_error:
3134             if not len(prs):
3135                 raise last_error
3136             self.report_warning(last_error)
3137         return prs, player_url
3138
3139     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, is_live, duration):
3140         itags, stream_ids = {}, []
3141         itag_qualities, res_qualities = {}, {}
3142         q = qualities([
3143             # Normally tiny is the smallest video-only formats. But
3144             # audio-only formats with unknown quality may get tagged as tiny
3145             'tiny',
3146             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3147             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3148         ])
3149         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3150
3151         for fmt in streaming_formats:
3152             if fmt.get('targetDurationSec'):
3153                 continue
3154
3155             itag = str_or_none(fmt.get('itag'))
3156             audio_track = fmt.get('audioTrack') or {}
3157             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3158             if stream_id in stream_ids:
3159                 continue
3160
3161             quality = fmt.get('quality')
3162             height = int_or_none(fmt.get('height'))
3163             if quality == 'tiny' or not quality:
3164                 quality = fmt.get('audioQuality', '').lower() or quality
3165             # The 3gp format (17) in android client has a quality of "small",
3166             # but is actually worse than other formats
3167             if itag == '17':
3168                 quality = 'tiny'
3169             if quality:
3170                 if itag:
3171                     itag_qualities[itag] = quality
3172                 if height:
3173                     res_qualities[height] = quality
3174             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3175             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3176             # number of fragment that would subsequently requested with (`&sq=N`)
3177             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3178                 continue
3179
3180             fmt_url = fmt.get('url')
3181             if not fmt_url:
3182                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3183                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3184                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3185                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3186                     continue
3187                 try:
3188                     fmt_url += '&%s=%s' % (
3189                         traverse_obj(sc, ('sp', -1)) or 'signature',
3190                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3191                     )
3192                 except ExtractorError as e:
3193                     self.report_warning('Signature extraction failed: Some formats may be missing', only_once=True)
3194                     self.write_debug(e, only_once=True)
3195                     continue
3196
3197             query = parse_qs(fmt_url)
3198             throttled = False
3199             if query.get('n'):
3200                 try:
3201                     fmt_url = update_url_query(fmt_url, {
3202                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3203                 except ExtractorError as e:
3204                     self.report_warning(
3205                         'nsig extraction failed: You may experience throttling for some formats\n'
3206                         f'n = {query["n"][0]} ; player = {player_url}', only_once=True)
3207                     self.write_debug(e, only_once=True)
3208                     throttled = True
3209
3210             if itag:
3211                 itags[itag] = 'https'
3212                 stream_ids.append(stream_id)
3213
3214             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3215             language_preference = (
3216                 10 if audio_track.get('audioIsDefault') and 10
3217                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3218                 else -1)
3219             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3220             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3221             # Make sure to avoid false positives with small duration differences.
3222             # Eg: __2ABJjxzNo, ySuUZEjARPY
3223             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3224             if is_damaged:
3225                 self.report_warning(
3226                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3227             dct = {
3228                 'asr': int_or_none(fmt.get('audioSampleRate')),
3229                 'filesize': int_or_none(fmt.get('contentLength')),
3230                 'format_id': itag,
3231                 'format_note': join_nonempty(
3232                     '%s%s' % (audio_track.get('displayName') or '',
3233                               ' (default)' if language_preference > 0 else ''),
3234                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3235                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3236                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3237                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3238                 'fps': int_or_none(fmt.get('fps')) or None,
3239                 'height': height,
3240                 'quality': q(quality),
3241                 'has_drm': bool(fmt.get('drmFamilies')),
3242                 'tbr': tbr,
3243                 'url': fmt_url,
3244                 'width': int_or_none(fmt.get('width')),
3245                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3246                                           'desc' if language_preference < -1 else ''),
3247                 'language_preference': language_preference,
3248                 # Strictly de-prioritize damaged and 3gp formats
3249                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3250             }
3251             mime_mobj = re.match(
3252                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3253             if mime_mobj:
3254                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3255                 dct.update(parse_codecs(mime_mobj.group(2)))
3256             no_audio = dct.get('acodec') == 'none'
3257             no_video = dct.get('vcodec') == 'none'
3258             if no_audio:
3259                 dct['vbr'] = tbr
3260             if no_video:
3261                 dct['abr'] = tbr
3262             if no_audio or no_video:
3263                 dct['downloader_options'] = {
3264                     # Youtube throttles chunks >~10M
3265                     'http_chunk_size': 10485760,
3266                 }
3267                 if dct.get('ext'):
3268                     dct['container'] = dct['ext'] + '_dash'
3269             yield dct
3270
3271         live_from_start = is_live and self.get_param('live_from_start')
3272         skip_manifests = self._configuration_arg('skip')
3273         if not self.get_param('youtube_include_hls_manifest', True):
3274             skip_manifests.append('hls')
3275         if not self.get_param('youtube_include_dash_manifest', True):
3276             skip_manifests.append('dash')
3277         get_dash = 'dash' not in skip_manifests and (
3278             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3279         get_hls = not live_from_start and 'hls' not in skip_manifests
3280
3281         def process_manifest_format(f, proto, itag):
3282             if itag in itags:
3283                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3284                     return False
3285                 itag = f'{itag}-{proto}'
3286             if itag:
3287                 f['format_id'] = itag
3288                 itags[itag] = proto
3289
3290             f['quality'] = next((
3291                 q(qdict[val])
3292                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3293                 if val in qdict), -1)
3294             return True
3295
3296         subtitles = {}
3297         for sd in streaming_data:
3298             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3299             if hls_manifest_url:
3300                 fmts, subs = self._extract_m3u8_formats_and_subtitles(hls_manifest_url, video_id, 'mp4', fatal=False, live=is_live)
3301                 subtitles = self._merge_subtitles(subs, subtitles)
3302                 for f in fmts:
3303                     if process_manifest_format(f, 'hls', self._search_regex(
3304                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3305                         yield f
3306
3307             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3308             if dash_manifest_url:
3309                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3310                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3311                 for f in formats:
3312                     if process_manifest_format(f, 'dash', f['format_id']):
3313                         f['filesize'] = int_or_none(self._search_regex(
3314                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3315                         if live_from_start:
3316                             f['is_from_start'] = True
3317
3318                         yield f
3319         yield subtitles
3320
3321     def _extract_storyboard(self, player_responses, duration):
3322         spec = get_first(
3323             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3324         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3325         if not base_url:
3326             return
3327         L = len(spec) - 1
3328         for i, args in enumerate(spec):
3329             args = args.split('#')
3330             counts = list(map(int_or_none, args[:5]))
3331             if len(args) != 8 or not all(counts):
3332                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3333                 continue
3334             width, height, frame_count, cols, rows = counts
3335             N, sigh = args[6:]
3336
3337             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3338             fragment_count = frame_count / (cols * rows)
3339             fragment_duration = duration / fragment_count
3340             yield {
3341                 'format_id': f'sb{i}',
3342                 'format_note': 'storyboard',
3343                 'ext': 'mhtml',
3344                 'protocol': 'mhtml',
3345                 'acodec': 'none',
3346                 'vcodec': 'none',
3347                 'url': url,
3348                 'width': width,
3349                 'height': height,
3350                 'fps': frame_count / duration,
3351                 'rows': rows,
3352                 'columns': cols,
3353                 'fragments': [{
3354                     'url': url.replace('$M', str(j)),
3355                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3356                 } for j in range(math.ceil(fragment_count))],
3357             }
3358
3359     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3360         webpage = None
3361         if 'webpage' not in self._configuration_arg('player_skip'):
3362             webpage = self._download_webpage(
3363                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3364
3365         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3366
3367         player_responses, player_url = self._extract_player_responses(
3368             self._get_requested_clients(url, smuggled_data),
3369             video_id, webpage, master_ytcfg)
3370
3371         return webpage, master_ytcfg, player_responses, player_url
3372
3373     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3374         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3375         is_live = get_first(video_details, 'isLive')
3376         if is_live is None:
3377             is_live = get_first(live_broadcast_details, 'isLiveNow')
3378
3379         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3380         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, is_live, duration)
3381
3382         return live_broadcast_details, is_live, streaming_data, formats, subtitles
3383
3384     def _real_extract(self, url):
3385         url, smuggled_data = unsmuggle_url(url, {})
3386         video_id = self._match_id(url)
3387
3388         base_url = self.http_scheme() + '//www.youtube.com/'
3389         webpage_url = base_url + 'watch?v=' + video_id
3390
3391         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3392
3393         playability_statuses = traverse_obj(
3394             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3395
3396         trailer_video_id = get_first(
3397             playability_statuses,
3398             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3399             expected_type=str)
3400         if trailer_video_id:
3401             return self.url_result(
3402                 trailer_video_id, self.ie_key(), trailer_video_id)
3403
3404         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3405                        if webpage else (lambda x: None))
3406
3407         video_details = traverse_obj(
3408             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3409         microformats = traverse_obj(
3410             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3411             expected_type=dict, default=[])
3412         video_title = (
3413             get_first(video_details, 'title')
3414             or self._get_text(microformats, (..., 'title'))
3415             or search_meta(['og:title', 'twitter:title', 'title']))
3416         video_description = get_first(video_details, 'shortDescription')
3417
3418         multifeed_metadata_list = get_first(
3419             player_responses,
3420             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3421             expected_type=str)
3422         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3423             if self.get_param('noplaylist'):
3424                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3425             else:
3426                 entries = []
3427                 feed_ids = []
3428                 for feed in multifeed_metadata_list.split(','):
3429                     # Unquote should take place before split on comma (,) since textual
3430                     # fields may contain comma as well (see
3431                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3432                     feed_data = urllib.parse.parse_qs(
3433                         urllib.parse.unquote_plus(feed))
3434
3435                     def feed_entry(name):
3436                         return try_get(
3437                             feed_data, lambda x: x[name][0], str)
3438
3439                     feed_id = feed_entry('id')
3440                     if not feed_id:
3441                         continue
3442                     feed_title = feed_entry('title')
3443                     title = video_title
3444                     if feed_title:
3445                         title += ' (%s)' % feed_title
3446                     entries.append({
3447                         '_type': 'url_transparent',
3448                         'ie_key': 'Youtube',
3449                         'url': smuggle_url(
3450                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3451                             {'force_singlefeed': True}),
3452                         'title': title,
3453                     })
3454                     feed_ids.append(feed_id)
3455                 self.to_screen(
3456                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3457                     % (', '.join(feed_ids), video_id))
3458                 return self.playlist_result(
3459                     entries, video_id, video_title, video_description)
3460
3461         duration = int_or_none(
3462             get_first(video_details, 'lengthSeconds')
3463             or get_first(microformats, 'lengthSeconds')
3464             or parse_duration(search_meta('duration'))) or None
3465
3466         if get_first(video_details, 'isPostLiveDvr'):
3467             self.write_debug('Video is in Post-Live Manifestless mode')
3468             if (duration or 0) > 4 * 3600:
3469                 self.report_warning(
3470                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3471                     'This is a known issue and patches are welcome')
3472
3473         live_broadcast_details, is_live, streaming_data, formats, automatic_captions = \
3474             self._list_formats(video_id, microformats, video_details, player_responses, player_url)
3475
3476         if not formats:
3477             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3478                 self.report_drm(video_id)
3479             pemr = get_first(
3480                 playability_statuses,
3481                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3482             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3483             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3484             if subreason:
3485                 if subreason == 'The uploader has not made this video available in your country.':
3486                     countries = get_first(microformats, 'availableCountries')
3487                     if not countries:
3488                         regions_allowed = search_meta('regionsAllowed')
3489                         countries = regions_allowed.split(',') if regions_allowed else None
3490                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3491                 reason += f'. {subreason}'
3492             if reason:
3493                 self.raise_no_formats(reason, expected=True)
3494
3495         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3496         if not keywords and webpage:
3497             keywords = [
3498                 unescapeHTML(m.group('content'))
3499                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3500         for keyword in keywords:
3501             if keyword.startswith('yt:stretch='):
3502                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3503                 if mobj:
3504                     # NB: float is intentional for forcing float division
3505                     w, h = (float(v) for v in mobj.groups())
3506                     if w > 0 and h > 0:
3507                         ratio = w / h
3508                         for f in formats:
3509                             if f.get('vcodec') != 'none':
3510                                 f['stretched_ratio'] = ratio
3511                         break
3512         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3513         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3514         if thumbnail_url:
3515             thumbnails.append({
3516                 'url': thumbnail_url,
3517             })
3518         original_thumbnails = thumbnails.copy()
3519
3520         # The best resolution thumbnails sometimes does not appear in the webpage
3521         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3522         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3523         thumbnail_names = [
3524             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3525             # in resolution, these are not the custom thumbnail. So de-prioritize them
3526             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3527             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3528         ]
3529         n_thumbnail_names = len(thumbnail_names)
3530         thumbnails.extend({
3531             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3532                 video_id=video_id, name=name, ext=ext,
3533                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3534         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3535         for thumb in thumbnails:
3536             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3537             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3538         self._remove_duplicate_formats(thumbnails)
3539         self._downloader._sort_thumbnails(original_thumbnails)
3540
3541         category = get_first(microformats, 'category') or search_meta('genre')
3542         channel_id = str_or_none(
3543             get_first(video_details, 'channelId')
3544             or get_first(microformats, 'externalChannelId')
3545             or search_meta('channelId'))
3546         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3547
3548         live_content = get_first(video_details, 'isLiveContent')
3549         is_upcoming = get_first(video_details, 'isUpcoming')
3550         if is_live is None:
3551             if is_upcoming or live_content is False:
3552                 is_live = False
3553         if is_upcoming is None and (live_content or is_live):
3554             is_upcoming = False
3555         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3556         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3557         if not duration and live_end_time and live_start_time:
3558             duration = live_end_time - live_start_time
3559
3560         if is_live and self.get_param('live_from_start'):
3561             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3562
3563         formats.extend(self._extract_storyboard(player_responses, duration))
3564
3565         # source_preference is lower for throttled/potentially damaged formats
3566         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3567
3568         info = {
3569             'id': video_id,
3570             'title': video_title,
3571             'formats': formats,
3572             'thumbnails': thumbnails,
3573             # The best thumbnail that we are sure exists. Prevents unnecessary
3574             # URL checking if user don't care about getting the best possible thumbnail
3575             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3576             'description': video_description,
3577             'uploader': get_first(video_details, 'author'),
3578             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3579             'uploader_url': owner_profile_url,
3580             'channel_id': channel_id,
3581             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3582             'duration': duration,
3583             'view_count': int_or_none(
3584                 get_first((video_details, microformats), (..., 'viewCount'))
3585                 or search_meta('interactionCount')),
3586             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3587             'age_limit': 18 if (
3588                 get_first(microformats, 'isFamilySafe') is False
3589                 or search_meta('isFamilyFriendly') == 'false'
3590                 or search_meta('og:restrictions:age') == '18+') else 0,
3591             'webpage_url': webpage_url,
3592             'categories': [category] if category else None,
3593             'tags': keywords,
3594             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3595             'is_live': is_live,
3596             'was_live': (False if is_live or is_upcoming or live_content is False
3597                          else None if is_live is None or is_upcoming is None
3598                          else live_content),
3599             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3600             'release_timestamp': live_start_time,
3601         }
3602
3603         subtitles = {}
3604         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3605         if pctr:
3606             def get_lang_code(track):
3607                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3608                         or track.get('languageCode'))
3609
3610             # Converted into dicts to remove duplicates
3611             captions = {
3612                 get_lang_code(sub): sub
3613                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3614             translation_languages = {
3615                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3616                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3617
3618             def process_language(container, base_url, lang_code, sub_name, query):
3619                 lang_subs = container.setdefault(lang_code, [])
3620                 for fmt in self._SUBTITLE_FORMATS:
3621                     query.update({
3622                         'fmt': fmt,
3623                     })
3624                     lang_subs.append({
3625                         'ext': fmt,
3626                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3627                         'name': sub_name,
3628                     })
3629
3630             # NB: Constructing the full subtitle dictionary is slow
3631             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
3632                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
3633             for lang_code, caption_track in captions.items():
3634                 base_url = caption_track.get('baseUrl')
3635                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3636                 if not base_url:
3637                     continue
3638                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3639                 if caption_track.get('kind') != 'asr':
3640                     if not lang_code:
3641                         continue
3642                     process_language(
3643                         subtitles, base_url, lang_code, lang_name, {})
3644                     if not caption_track.get('isTranslatable'):
3645                         continue
3646                 for trans_code, trans_name in translation_languages.items():
3647                     if not trans_code:
3648                         continue
3649                     orig_trans_code = trans_code
3650                     if caption_track.get('kind') != 'asr':
3651                         if not get_translated_subs:
3652                             continue
3653                         trans_code += f'-{lang_code}'
3654                         trans_name += format_field(lang_name, None, ' from %s')
3655                     # Add an "-orig" label to the original language so that it can be distinguished.
3656                     # The subs are returned without "-orig" as well for compatibility
3657                     if lang_code == f'a-{orig_trans_code}':
3658                         process_language(
3659                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3660                     # Setting tlang=lang returns damaged subtitles.
3661                     process_language(automatic_captions, base_url, trans_code, trans_name,
3662                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3663
3664         info['automatic_captions'] = automatic_captions
3665         info['subtitles'] = subtitles
3666
3667         parsed_url = urllib.parse.urlparse(url)
3668         for component in [parsed_url.fragment, parsed_url.query]:
3669             query = urllib.parse.parse_qs(component)
3670             for k, v in query.items():
3671                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3672                     d_k += '_time'
3673                     if d_k not in info and k in s_ks:
3674                         info[d_k] = parse_duration(query[k][0])
3675
3676         # Youtube Music Auto-generated description
3677         if video_description:
3678             mobj = re.search(
3679                 r'''(?xs)
3680                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3681                     (?P<album>[^\n]+)
3682                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3683                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3684                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3685                     .+\nAuto-generated\ by\ YouTube\.\s*$
3686                 ''', video_description)
3687             if mobj:
3688                 release_year = mobj.group('release_year')
3689                 release_date = mobj.group('release_date')
3690                 if release_date:
3691                     release_date = release_date.replace('-', '')
3692                     if not release_year:
3693                         release_year = release_date[:4]
3694                 info.update({
3695                     'album': mobj.group('album'.strip()),
3696                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3697                     'track': mobj.group('track').strip(),
3698                     'release_date': release_date,
3699                     'release_year': int_or_none(release_year),
3700                 })
3701
3702         initial_data = None
3703         if webpage:
3704             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
3705         if not initial_data:
3706             query = {'videoId': video_id}
3707             query.update(self._get_checkok_params())
3708             initial_data = self._extract_response(
3709                 item_id=video_id, ep='next', fatal=False,
3710                 ytcfg=master_ytcfg, query=query,
3711                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3712                 note='Downloading initial data API JSON')
3713
3714         info['comment_count'] = traverse_obj(initial_data, (
3715             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
3716             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
3717         ), (
3718             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
3719             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
3720         ), expected_type=int_or_none, get_all=False)
3721
3722         try:  # This will error if there is no livechat
3723             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3724         except (KeyError, IndexError, TypeError):
3725             pass
3726         else:
3727             info.setdefault('subtitles', {})['live_chat'] = [{
3728                 # url is needed to set cookies
3729                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
3730                 'video_id': video_id,
3731                 'ext': 'json',
3732                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3733             }]
3734
3735         if initial_data:
3736             info['chapters'] = (
3737                 self._extract_chapters_from_json(initial_data, duration)
3738                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3739                 or self._extract_chapters_from_description(video_description, duration)
3740                 or None)
3741
3742         contents = traverse_obj(
3743             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3744             expected_type=list, default=[])
3745
3746         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3747         if vpir:
3748             stl = vpir.get('superTitleLink')
3749             if stl:
3750                 stl = self._get_text(stl)
3751                 if try_get(
3752                         vpir,
3753                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3754                     info['location'] = stl
3755                 else:
3756                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3757                     if mobj:
3758                         info.update({
3759                             'series': mobj.group(1),
3760                             'season_number': int(mobj.group(2)),
3761                             'episode_number': int(mobj.group(3)),
3762                         })
3763             for tlb in (try_get(
3764                     vpir,
3765                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3766                     list) or []):
3767                 tbr = tlb.get('toggleButtonRenderer') or {}
3768                 for getter, regex in [(
3769                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3770                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3771                             lambda x: x['accessibility'],
3772                             lambda x: x['accessibilityData']['accessibilityData'],
3773                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3774                     label = (try_get(tbr, getter, dict) or {}).get('label')
3775                     if label:
3776                         mobj = re.match(regex, label)
3777                         if mobj:
3778                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3779                             break
3780             sbr_tooltip = try_get(
3781                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3782             if sbr_tooltip:
3783                 like_count, dislike_count = sbr_tooltip.split(' / ')
3784                 info.update({
3785                     'like_count': str_to_int(like_count),
3786                     'dislike_count': str_to_int(dislike_count),
3787                 })
3788         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3789         if vsir:
3790             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3791             info.update({
3792                 'channel': self._get_text(vor, 'title'),
3793                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3794
3795             rows = try_get(
3796                 vsir,
3797                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3798                 list) or []
3799             multiple_songs = False
3800             for row in rows:
3801                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3802                     multiple_songs = True
3803                     break
3804             for row in rows:
3805                 mrr = row.get('metadataRowRenderer') or {}
3806                 mrr_title = mrr.get('title')
3807                 if not mrr_title:
3808                     continue
3809                 mrr_title = self._get_text(mrr, 'title')
3810                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3811                 if mrr_title == 'License':
3812                     info['license'] = mrr_contents_text
3813                 elif not multiple_songs:
3814                     if mrr_title == 'Album':
3815                         info['album'] = mrr_contents_text
3816                     elif mrr_title == 'Artist':
3817                         info['artist'] = mrr_contents_text
3818                     elif mrr_title == 'Song':
3819                         info['track'] = mrr_contents_text
3820
3821         fallbacks = {
3822             'channel': 'uploader',
3823             'channel_id': 'uploader_id',
3824             'channel_url': 'uploader_url',
3825         }
3826
3827         # The upload date for scheduled, live and past live streams / premieres in microformats
3828         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3829         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3830         upload_date = (
3831             unified_strdate(get_first(microformats, 'uploadDate'))
3832             or unified_strdate(search_meta('uploadDate')))
3833         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3834             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3835         info['upload_date'] = upload_date
3836
3837         for to, frm in fallbacks.items():
3838             if not info.get(to):
3839                 info[to] = info.get(frm)
3840
3841         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3842             v = info.get(s_k)
3843             if v:
3844                 info[d_k] = v
3845
3846         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3847         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3848         is_membersonly = None
3849         is_premium = None
3850         if initial_data and is_private is not None:
3851             is_membersonly = False
3852             is_premium = False
3853             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3854             badge_labels = set()
3855             for content in contents:
3856                 if not isinstance(content, dict):
3857                     continue
3858                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3859             for badge_label in badge_labels:
3860                 if badge_label.lower() == 'members only':
3861                     is_membersonly = True
3862                 elif badge_label.lower() == 'premium':
3863                     is_premium = True
3864                 elif badge_label.lower() == 'unlisted':
3865                     is_unlisted = True
3866
3867         info['availability'] = self._availability(
3868             is_private=is_private,
3869             needs_premium=is_premium,
3870             needs_subscription=is_membersonly,
3871             needs_auth=info['age_limit'] >= 18,
3872             is_unlisted=None if is_private is None else is_unlisted)
3873
3874         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3875
3876         self.mark_watched(video_id, player_responses)
3877
3878         return info
3879
3880
3881 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3882
3883     @staticmethod
3884     def passthrough_smuggled_data(func):
3885         def _smuggle(entries, smuggled_data):
3886             for entry in entries:
3887                 # TODO: Convert URL to music.youtube instead.
3888                 # Do we need to passthrough any other smuggled_data?
3889                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3890                 yield entry
3891
3892         @functools.wraps(func)
3893         def wrapper(self, url):
3894             url, smuggled_data = unsmuggle_url(url, {})
3895             if self.is_music_url(url):
3896                 smuggled_data['is_music_url'] = True
3897             info_dict = func(self, url, smuggled_data)
3898             if smuggled_data and info_dict.get('entries'):
3899                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3900             return info_dict
3901         return wrapper
3902
3903     def _extract_channel_id(self, webpage):
3904         channel_id = self._html_search_meta(
3905             'channelId', webpage, 'channel id', default=None)
3906         if channel_id:
3907             return channel_id
3908         channel_url = self._html_search_meta(
3909             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3910              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3911              'twitter:app:url:googleplay'), webpage, 'channel url')
3912         return self._search_regex(
3913             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3914             channel_url, 'channel id')
3915
3916     @staticmethod
3917     def _extract_basic_item_renderer(item):
3918         # Modified from _extract_grid_item_renderer
3919         known_basic_renderers = (
3920             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3921         )
3922         for key, renderer in item.items():
3923             if not isinstance(renderer, dict):
3924                 continue
3925             elif key in known_basic_renderers:
3926                 return renderer
3927             elif key.startswith('grid') and key.endswith('Renderer'):
3928                 return renderer
3929
3930     def _grid_entries(self, grid_renderer):
3931         for item in grid_renderer['items']:
3932             if not isinstance(item, dict):
3933                 continue
3934             renderer = self._extract_basic_item_renderer(item)
3935             if not isinstance(renderer, dict):
3936                 continue
3937             title = self._get_text(renderer, 'title')
3938
3939             # playlist
3940             playlist_id = renderer.get('playlistId')
3941             if playlist_id:
3942                 yield self.url_result(
3943                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3944                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3945                     video_title=title)
3946                 continue
3947             # video
3948             video_id = renderer.get('videoId')
3949             if video_id:
3950                 yield self._extract_video(renderer)
3951                 continue
3952             # channel
3953             channel_id = renderer.get('channelId')
3954             if channel_id:
3955                 yield self.url_result(
3956                     'https://www.youtube.com/channel/%s' % channel_id,
3957                     ie=YoutubeTabIE.ie_key(), video_title=title)
3958                 continue
3959             # generic endpoint URL support
3960             ep_url = urljoin('https://www.youtube.com/', try_get(
3961                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3962                 str))
3963             if ep_url:
3964                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3965                     if ie.suitable(ep_url):
3966                         yield self.url_result(
3967                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3968                         break
3969
3970     def _music_reponsive_list_entry(self, renderer):
3971         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3972         if video_id:
3973             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3974                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3975         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3976         if playlist_id:
3977             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3978             if video_id:
3979                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3980                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3981             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3982                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3983         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3984         if browse_id:
3985             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3986                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3987
3988     def _shelf_entries_from_content(self, shelf_renderer):
3989         content = shelf_renderer.get('content')
3990         if not isinstance(content, dict):
3991             return
3992         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3993         if renderer:
3994             # TODO: add support for nested playlists so each shelf is processed
3995             # as separate playlist
3996             # TODO: this includes only first N items
3997             yield from self._grid_entries(renderer)
3998         renderer = content.get('horizontalListRenderer')
3999         if renderer:
4000             # TODO
4001             pass
4002
4003     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4004         ep = try_get(
4005             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4006             str)
4007         shelf_url = urljoin('https://www.youtube.com', ep)
4008         if shelf_url:
4009             # Skipping links to another channels, note that checking for
4010             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4011             # will not work
4012             if skip_channels and '/channels?' in shelf_url:
4013                 return
4014             title = self._get_text(shelf_renderer, 'title')
4015             yield self.url_result(shelf_url, video_title=title)
4016         # Shelf may not contain shelf URL, fallback to extraction from content
4017         yield from self._shelf_entries_from_content(shelf_renderer)
4018
4019     def _playlist_entries(self, video_list_renderer):
4020         for content in video_list_renderer['contents']:
4021             if not isinstance(content, dict):
4022                 continue
4023             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4024             if not isinstance(renderer, dict):
4025                 continue
4026             video_id = renderer.get('videoId')
4027             if not video_id:
4028                 continue
4029             yield self._extract_video(renderer)
4030
4031     def _rich_entries(self, rich_grid_renderer):
4032         renderer = try_get(
4033             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
4034         video_id = renderer.get('videoId')
4035         if not video_id:
4036             return
4037         yield self._extract_video(renderer)
4038
4039     def _video_entry(self, video_renderer):
4040         video_id = video_renderer.get('videoId')
4041         if video_id:
4042             return self._extract_video(video_renderer)
4043
4044     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4045         url = urljoin('https://youtube.com', traverse_obj(
4046             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4047         if url:
4048             return self.url_result(
4049                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4050
4051     def _post_thread_entries(self, post_thread_renderer):
4052         post_renderer = try_get(
4053             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4054         if not post_renderer:
4055             return
4056         # video attachment
4057         video_renderer = try_get(
4058             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4059         video_id = video_renderer.get('videoId')
4060         if video_id:
4061             entry = self._extract_video(video_renderer)
4062             if entry:
4063                 yield entry
4064         # playlist attachment
4065         playlist_id = try_get(
4066             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4067         if playlist_id:
4068             yield self.url_result(
4069                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4070                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4071         # inline video links
4072         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4073         for run in runs:
4074             if not isinstance(run, dict):
4075                 continue
4076             ep_url = try_get(
4077                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4078             if not ep_url:
4079                 continue
4080             if not YoutubeIE.suitable(ep_url):
4081                 continue
4082             ep_video_id = YoutubeIE._match_id(ep_url)
4083             if video_id == ep_video_id:
4084                 continue
4085             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4086
4087     def _post_thread_continuation_entries(self, post_thread_continuation):
4088         contents = post_thread_continuation.get('contents')
4089         if not isinstance(contents, list):
4090             return
4091         for content in contents:
4092             renderer = content.get('backstagePostThreadRenderer')
4093             if isinstance(renderer, dict):
4094                 yield from self._post_thread_entries(renderer)
4095                 continue
4096             renderer = content.get('videoRenderer')
4097             if isinstance(renderer, dict):
4098                 yield self._video_entry(renderer)
4099
4100     r''' # unused
4101     def _rich_grid_entries(self, contents):
4102         for content in contents:
4103             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4104             if video_renderer:
4105                 entry = self._video_entry(video_renderer)
4106                 if entry:
4107                     yield entry
4108     '''
4109
4110     def _extract_entries(self, parent_renderer, continuation_list):
4111         # continuation_list is modified in-place with continuation_list = [continuation_token]
4112         continuation_list[:] = [None]
4113         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4114         for content in contents:
4115             if not isinstance(content, dict):
4116                 continue
4117             is_renderer = traverse_obj(
4118                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4119                 expected_type=dict)
4120             if not is_renderer:
4121                 renderer = content.get('richItemRenderer')
4122                 if renderer:
4123                     for entry in self._rich_entries(renderer):
4124                         yield entry
4125                     continuation_list[0] = self._extract_continuation(parent_renderer)
4126                 continue
4127             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4128             for isr_content in isr_contents:
4129                 if not isinstance(isr_content, dict):
4130                     continue
4131
4132                 known_renderers = {
4133                     'playlistVideoListRenderer': self._playlist_entries,
4134                     'gridRenderer': self._grid_entries,
4135                     'reelShelfRenderer': self._grid_entries,
4136                     'shelfRenderer': self._shelf_entries,
4137                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4138                     'backstagePostThreadRenderer': self._post_thread_entries,
4139                     'videoRenderer': lambda x: [self._video_entry(x)],
4140                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4141                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4142                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4143                 }
4144                 for key, renderer in isr_content.items():
4145                     if key not in known_renderers:
4146                         continue
4147                     for entry in known_renderers[key](renderer):
4148                         if entry:
4149                             yield entry
4150                     continuation_list[0] = self._extract_continuation(renderer)
4151                     break
4152
4153             if not continuation_list[0]:
4154                 continuation_list[0] = self._extract_continuation(is_renderer)
4155
4156         if not continuation_list[0]:
4157             continuation_list[0] = self._extract_continuation(parent_renderer)
4158
4159     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4160         continuation_list = [None]
4161         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4162         tab_content = try_get(tab, lambda x: x['content'], dict)
4163         if not tab_content:
4164             return
4165         parent_renderer = (
4166             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4167             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4168         yield from extract_entries(parent_renderer)
4169         continuation = continuation_list[0]
4170
4171         for page_num in itertools.count(1):
4172             if not continuation:
4173                 break
4174             headers = self.generate_api_headers(
4175                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4176             response = self._extract_response(
4177                 item_id=f'{item_id} page {page_num}',
4178                 query=continuation, headers=headers, ytcfg=ytcfg,
4179                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4180
4181             if not response:
4182                 break
4183             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4184             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4185             visitor_data = self._extract_visitor_data(response) or visitor_data
4186
4187             known_continuation_renderers = {
4188                 'playlistVideoListContinuation': self._playlist_entries,
4189                 'gridContinuation': self._grid_entries,
4190                 'itemSectionContinuation': self._post_thread_continuation_entries,
4191                 'sectionListContinuation': extract_entries,  # for feeds
4192             }
4193             continuation_contents = try_get(
4194                 response, lambda x: x['continuationContents'], dict) or {}
4195             continuation_renderer = None
4196             for key, value in continuation_contents.items():
4197                 if key not in known_continuation_renderers:
4198                     continue
4199                 continuation_renderer = value
4200                 continuation_list = [None]
4201                 yield from known_continuation_renderers[key](continuation_renderer)
4202                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4203                 break
4204             if continuation_renderer:
4205                 continue
4206
4207             known_renderers = {
4208                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4209                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4210                 'gridVideoRenderer': (self._grid_entries, 'items'),
4211                 'gridChannelRenderer': (self._grid_entries, 'items'),
4212                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4213                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4214                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4215                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4216             }
4217             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4218             continuation_items = try_get(
4219                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4220             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4221             video_items_renderer = None
4222             for key, value in continuation_item.items():
4223                 if key not in known_renderers:
4224                     continue
4225                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4226                 continuation_list = [None]
4227                 yield from known_renderers[key][0](video_items_renderer)
4228                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4229                 break
4230             if video_items_renderer:
4231                 continue
4232             break
4233
4234     @staticmethod
4235     def _extract_selected_tab(tabs, fatal=True):
4236         for tab in tabs:
4237             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4238             if renderer.get('selected') is True:
4239                 return renderer
4240         else:
4241             if fatal:
4242                 raise ExtractorError('Unable to find selected tab')
4243
4244     def _extract_uploader(self, data):
4245         uploader = {}
4246         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4247         owner = try_get(
4248             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4249         if owner:
4250             owner_text = owner.get('text')
4251             uploader['uploader'] = self._search_regex(
4252                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4253             uploader['uploader_id'] = try_get(
4254                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], str)
4255             uploader['uploader_url'] = urljoin(
4256                 'https://www.youtube.com/',
4257                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], str))
4258         return {k: v for k, v in uploader.items() if v is not None}
4259
4260     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4261         playlist_id = title = description = channel_url = channel_name = channel_id = None
4262         tags = []
4263
4264         selected_tab = self._extract_selected_tab(tabs)
4265         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4266         renderer = try_get(
4267             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4268         if renderer:
4269             channel_name = renderer.get('title')
4270             channel_url = renderer.get('channelUrl')
4271             channel_id = renderer.get('externalId')
4272         else:
4273             renderer = try_get(
4274                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4275
4276         if renderer:
4277             title = renderer.get('title')
4278             description = renderer.get('description', '')
4279             playlist_id = channel_id
4280             tags = renderer.get('keywords', '').split()
4281
4282         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4283         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4284         def _get_uncropped(url):
4285             return url_or_none((url or '').split('=')[0] + '=s0')
4286
4287         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4288         if avatar_thumbnails:
4289             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4290             if uncropped_avatar:
4291                 avatar_thumbnails.append({
4292                     'url': uncropped_avatar,
4293                     'id': 'avatar_uncropped',
4294                     'preference': 1
4295                 })
4296
4297         channel_banners = self._extract_thumbnails(
4298             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4299         for banner in channel_banners:
4300             banner['preference'] = -10
4301
4302         if channel_banners:
4303             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4304             if uncropped_banner:
4305                 channel_banners.append({
4306                     'url': uncropped_banner,
4307                     'id': 'banner_uncropped',
4308                     'preference': -5
4309                 })
4310
4311         primary_thumbnails = self._extract_thumbnails(
4312             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4313
4314         if playlist_id is None:
4315             playlist_id = item_id
4316
4317         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4318         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4319         if title is None:
4320             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4321         title += format_field(selected_tab, 'title', ' - %s')
4322         title += format_field(selected_tab, 'expandedText', ' - %s')
4323
4324         metadata = {
4325             'playlist_id': playlist_id,
4326             'playlist_title': title,
4327             'playlist_description': description,
4328             'uploader': channel_name,
4329             'uploader_id': channel_id,
4330             'uploader_url': channel_url,
4331             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4332             'tags': tags,
4333             'view_count': self._get_count(playlist_stats, 1),
4334             'availability': self._extract_availability(data),
4335             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4336             'playlist_count': self._get_count(playlist_stats, 0),
4337             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4338         }
4339         if not channel_id:
4340             metadata.update(self._extract_uploader(data))
4341         metadata.update({
4342             'channel': metadata['uploader'],
4343             'channel_id': metadata['uploader_id'],
4344             'channel_url': metadata['uploader_url']})
4345         return self.playlist_result(
4346             self._entries(
4347                 selected_tab, playlist_id, ytcfg,
4348                 self._extract_account_syncid(ytcfg, data),
4349                 self._extract_visitor_data(data, ytcfg)),
4350             **metadata)
4351
4352     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4353         first_id = last_id = response = None
4354         for page_num in itertools.count(1):
4355             videos = list(self._playlist_entries(playlist))
4356             if not videos:
4357                 return
4358             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4359             if start >= len(videos):
4360                 return
4361             yield from videos[start:]
4362             first_id = first_id or videos[0]['id']
4363             last_id = videos[-1]['id']
4364             watch_endpoint = try_get(
4365                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4366             headers = self.generate_api_headers(
4367                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4368                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4369             query = {
4370                 'playlistId': playlist_id,
4371                 'videoId': watch_endpoint.get('videoId') or last_id,
4372                 'index': watch_endpoint.get('index') or len(videos),
4373                 'params': watch_endpoint.get('params') or 'OAE%3D'
4374             }
4375             response = self._extract_response(
4376                 item_id='%s page %d' % (playlist_id, page_num),
4377                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4378                 check_get_keys='contents'
4379             )
4380             playlist = try_get(
4381                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4382
4383     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4384         title = playlist.get('title') or try_get(
4385             data, lambda x: x['titleText']['simpleText'], str)
4386         playlist_id = playlist.get('playlistId') or item_id
4387
4388         # Delegating everything except mix playlists to regular tab-based playlist URL
4389         playlist_url = urljoin(url, try_get(
4390             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4391             str))
4392
4393         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4394         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4395         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4396
4397         if playlist_url and playlist_url != url and not is_known_unviewable:
4398             return self.url_result(
4399                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4400                 video_title=title)
4401
4402         return self.playlist_result(
4403             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4404             playlist_id=playlist_id, playlist_title=title)
4405
4406     def _extract_availability(self, data):
4407         """
4408         Gets the availability of a given playlist/tab.
4409         Note: Unless YouTube tells us explicitly, we do not assume it is public
4410         @param data: response
4411         """
4412         is_private = is_unlisted = None
4413         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4414         badge_labels = self._extract_badges(renderer)
4415
4416         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4417         privacy_dropdown_entries = try_get(
4418             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4419         for renderer_dict in privacy_dropdown_entries:
4420             is_selected = try_get(
4421                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4422             if not is_selected:
4423                 continue
4424             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4425             if label:
4426                 badge_labels.add(label.lower())
4427                 break
4428
4429         for badge_label in badge_labels:
4430             if badge_label == 'unlisted':
4431                 is_unlisted = True
4432             elif badge_label == 'private':
4433                 is_private = True
4434             elif badge_label == 'public':
4435                 is_unlisted = is_private = False
4436         return self._availability(is_private, False, False, False, is_unlisted)
4437
4438     @staticmethod
4439     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4440         sidebar_renderer = try_get(
4441             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4442         for item in sidebar_renderer:
4443             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4444             if renderer:
4445                 return renderer
4446
4447     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4448         """
4449         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4450         """
4451         browse_id = params = None
4452         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4453         if not renderer:
4454             return
4455         menu_renderer = try_get(
4456             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4457         for menu_item in menu_renderer:
4458             if not isinstance(menu_item, dict):
4459                 continue
4460             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4461             text = try_get(
4462                 nav_item_renderer, lambda x: x['text']['simpleText'], str)
4463             if not text or text.lower() != 'show unavailable videos':
4464                 continue
4465             browse_endpoint = try_get(
4466                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4467             browse_id = browse_endpoint.get('browseId')
4468             params = browse_endpoint.get('params')
4469             break
4470
4471         headers = self.generate_api_headers(
4472             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4473             visitor_data=self._extract_visitor_data(data, ytcfg))
4474         query = {
4475             'params': params or 'wgYCCAA=',
4476             'browseId': browse_id or 'VL%s' % item_id
4477         }
4478         return self._extract_response(
4479             item_id=item_id, headers=headers, query=query,
4480             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4481             note='Downloading API JSON with unavailable videos')
4482
4483     @functools.cached_property
4484     def skip_webpage(self):
4485         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4486
4487     def _extract_webpage(self, url, item_id, fatal=True):
4488         retries = self.get_param('extractor_retries', 3)
4489         count = -1
4490         webpage = data = last_error = None
4491         while count < retries:
4492             count += 1
4493             # Sometimes youtube returns a webpage with incomplete ytInitialData
4494             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4495             if last_error:
4496                 self.report_warning('%s. Retrying ...' % last_error)
4497             try:
4498                 webpage = self._download_webpage(
4499                     url, item_id,
4500                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4501                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4502             except ExtractorError as e:
4503                 if isinstance(e.cause, network_exceptions):
4504                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4505                         last_error = error_to_compat_str(e.cause or e.msg)
4506                         if count < retries:
4507                             continue
4508                 if fatal:
4509                     raise
4510                 self.report_warning(error_to_compat_str(e))
4511                 break
4512             else:
4513                 try:
4514                     self._extract_and_report_alerts(data)
4515                 except ExtractorError as e:
4516                     if fatal:
4517                         raise
4518                     self.report_warning(error_to_compat_str(e))
4519                     break
4520
4521                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4522                     break
4523
4524                 last_error = 'Incomplete yt initial data received'
4525                 if count >= retries:
4526                     if fatal:
4527                         raise ExtractorError(last_error)
4528                     self.report_warning(last_error)
4529                     break
4530
4531         return webpage, data
4532
4533     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4534         """Use if failed to extract ytcfg (and data) from initial webpage"""
4535         if not ytcfg and self.is_authenticated:
4536             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4537             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4538                 raise ExtractorError(
4539                     f'{msg}. If you are not downloading private content, or '
4540                     'your cookies are only for the first account and channel,'
4541                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4542                     expected=True)
4543             self.report_warning(msg, only_once=True)
4544
4545     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4546         data = None
4547         if not self.skip_webpage:
4548             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4549             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4550             # Reject webpage data if redirected to home page without explicitly requesting
4551             selected_tab = self._extract_selected_tab(traverse_obj(
4552                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4553             if (url != 'https://www.youtube.com/feed/recommended'
4554                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4555                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4556                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4557                 if fatal:
4558                     raise ExtractorError(msg, expected=True)
4559                 self.report_warning(msg, only_once=True)
4560         if not data:
4561             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4562             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4563         return data, ytcfg
4564
4565     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4566         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4567         resolve_response = self._extract_response(
4568             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4569             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4570         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4571         for ep_key, ep in endpoints.items():
4572             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4573             if params:
4574                 return self._extract_response(
4575                     item_id=item_id, query=params, ep=ep, headers=headers,
4576                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4577                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4578         err_note = 'Failed to resolve url (does the playlist exist?)'
4579         if fatal:
4580             raise ExtractorError(err_note, expected=True)
4581         self.report_warning(err_note, item_id)
4582
4583     _SEARCH_PARAMS = None
4584
4585     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4586         data = {'query': query}
4587         if params is NO_DEFAULT:
4588             params = self._SEARCH_PARAMS
4589         if params:
4590             data['params'] = params
4591
4592         content_keys = (
4593             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4594             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4595             # ytmusic search
4596             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4597             ('continuationContents', ),
4598         )
4599         display_id = f'query "{query}"'
4600         check_get_keys = tuple({keys[0] for keys in content_keys})
4601         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4602         self._report_playlist_authcheck(ytcfg, fatal=False)
4603
4604         continuation_list = [None]
4605         search = None
4606         for page_num in itertools.count(1):
4607             data.update(continuation_list[0] or {})
4608             headers = self.generate_api_headers(
4609                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4610             search = self._extract_response(
4611                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4612                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4613             slr_contents = traverse_obj(search, *content_keys)
4614             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4615             if not continuation_list[0]:
4616                 break
4617
4618
4619 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4620     IE_DESC = 'YouTube Tabs'
4621     _VALID_URL = r'''(?x:
4622         https?://
4623             (?:\w+\.)?
4624             (?:
4625                 youtube(?:kids)?\.com|
4626                 %(invidious)s
4627             )/
4628             (?:
4629                 (?P<channel_type>channel|c|user|browse)/|
4630                 (?P<not_channel>
4631                     feed/|hashtag/|
4632                     (?:playlist|watch)\?.*?\blist=
4633                 )|
4634                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4635             )
4636             (?P<id>[^/?\#&]+)
4637     )''' % {
4638         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4639         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4640     }
4641     IE_NAME = 'youtube:tab'
4642
4643     _TESTS = [{
4644         'note': 'playlists, multipage',
4645         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4646         'playlist_mincount': 94,
4647         'info_dict': {
4648             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4649             'title': 'Igor Kleiner - Playlists',
4650             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4651             'uploader': 'Igor Kleiner',
4652             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4653             'channel': 'Igor Kleiner',
4654             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4655             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4656             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4657             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4658             'channel_follower_count': int
4659         },
4660     }, {
4661         'note': 'playlists, multipage, different order',
4662         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4663         'playlist_mincount': 94,
4664         'info_dict': {
4665             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4666             'title': 'Igor Kleiner - Playlists',
4667             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4668             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4669             'uploader': 'Igor Kleiner',
4670             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4671             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4672             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4673             'channel': 'Igor Kleiner',
4674             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4675             'channel_follower_count': int
4676         },
4677     }, {
4678         'note': 'playlists, series',
4679         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4680         'playlist_mincount': 5,
4681         'info_dict': {
4682             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4683             'title': '3Blue1Brown - Playlists',
4684             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4685             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4686             'uploader': '3Blue1Brown',
4687             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4688             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4689             'channel': '3Blue1Brown',
4690             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4691             'tags': ['Mathematics'],
4692             'channel_follower_count': int
4693         },
4694     }, {
4695         'note': 'playlists, singlepage',
4696         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4697         'playlist_mincount': 4,
4698         'info_dict': {
4699             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4700             'title': 'ThirstForScience - Playlists',
4701             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4702             'uploader': 'ThirstForScience',
4703             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4704             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4705             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4706             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4707             'tags': 'count:13',
4708             'channel': 'ThirstForScience',
4709             'channel_follower_count': int
4710         }
4711     }, {
4712         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4713         'only_matching': True,
4714     }, {
4715         'note': 'basic, single video playlist',
4716         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4717         'info_dict': {
4718             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4719             'uploader': 'Sergey M.',
4720             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4721             'title': 'youtube-dl public playlist',
4722             'description': '',
4723             'tags': [],
4724             'view_count': int,
4725             'modified_date': '20201130',
4726             'channel': 'Sergey M.',
4727             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4728             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4729             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4730         },
4731         'playlist_count': 1,
4732     }, {
4733         'note': 'empty playlist',
4734         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4735         'info_dict': {
4736             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4737             'uploader': 'Sergey M.',
4738             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4739             'title': 'youtube-dl empty playlist',
4740             'tags': [],
4741             'channel': 'Sergey M.',
4742             'description': '',
4743             'modified_date': '20160902',
4744             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4745             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4746             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4747         },
4748         'playlist_count': 0,
4749     }, {
4750         'note': 'Home tab',
4751         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4752         'info_dict': {
4753             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4754             'title': 'lex will - Home',
4755             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4756             'uploader': 'lex will',
4757             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4758             'channel': 'lex will',
4759             'tags': ['bible', 'history', 'prophesy'],
4760             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4761             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4762             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4763             'channel_follower_count': int
4764         },
4765         'playlist_mincount': 2,
4766     }, {
4767         'note': 'Videos tab',
4768         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4769         'info_dict': {
4770             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4771             'title': 'lex will - Videos',
4772             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4773             'uploader': 'lex will',
4774             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4775             'tags': ['bible', 'history', 'prophesy'],
4776             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4777             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4778             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4779             'channel': 'lex will',
4780             'channel_follower_count': int
4781         },
4782         'playlist_mincount': 975,
4783     }, {
4784         'note': 'Videos tab, sorted by popular',
4785         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4786         'info_dict': {
4787             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4788             'title': 'lex will - Videos',
4789             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4790             'uploader': 'lex will',
4791             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4792             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4793             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4794             'channel': 'lex will',
4795             'tags': ['bible', 'history', 'prophesy'],
4796             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4797             'channel_follower_count': int
4798         },
4799         'playlist_mincount': 199,
4800     }, {
4801         'note': 'Playlists tab',
4802         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4803         'info_dict': {
4804             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4805             'title': 'lex will - Playlists',
4806             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4807             'uploader': 'lex will',
4808             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4809             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4810             'channel': 'lex will',
4811             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4812             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4813             'tags': ['bible', 'history', 'prophesy'],
4814             'channel_follower_count': int
4815         },
4816         'playlist_mincount': 17,
4817     }, {
4818         'note': 'Community tab',
4819         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4820         'info_dict': {
4821             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4822             'title': 'lex will - Community',
4823             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4824             'uploader': 'lex will',
4825             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4826             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4827             'channel': 'lex will',
4828             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4829             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4830             'tags': ['bible', 'history', 'prophesy'],
4831             'channel_follower_count': int
4832         },
4833         'playlist_mincount': 18,
4834     }, {
4835         'note': 'Channels tab',
4836         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4837         'info_dict': {
4838             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4839             'title': 'lex will - Channels',
4840             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4841             'uploader': 'lex will',
4842             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4843             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4844             'channel': 'lex will',
4845             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4846             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4847             'tags': ['bible', 'history', 'prophesy'],
4848             'channel_follower_count': int
4849         },
4850         'playlist_mincount': 12,
4851     }, {
4852         'note': 'Search tab',
4853         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4854         'playlist_mincount': 40,
4855         'info_dict': {
4856             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4857             'title': '3Blue1Brown - Search - linear algebra',
4858             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4859             'uploader': '3Blue1Brown',
4860             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4861             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4862             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4863             'tags': ['Mathematics'],
4864             'channel': '3Blue1Brown',
4865             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4866             'channel_follower_count': int
4867         },
4868     }, {
4869         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4870         'only_matching': True,
4871     }, {
4872         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4873         'only_matching': True,
4874     }, {
4875         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4876         'only_matching': True,
4877     }, {
4878         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4879         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4880         'info_dict': {
4881             'title': '29C3: Not my department',
4882             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4883             'uploader': 'Christiaan008',
4884             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4885             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4886             'tags': [],
4887             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4888             'view_count': int,
4889             'modified_date': '20150605',
4890             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4891             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4892             'channel': 'Christiaan008',
4893         },
4894         'playlist_count': 96,
4895     }, {
4896         'note': 'Large playlist',
4897         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4898         'info_dict': {
4899             'title': 'Uploads from Cauchemar',
4900             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4901             'uploader': 'Cauchemar',
4902             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4903             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4904             'tags': [],
4905             'modified_date': r're:\d{8}',
4906             'channel': 'Cauchemar',
4907             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4908             'view_count': int,
4909             'description': '',
4910             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4911         },
4912         'playlist_mincount': 1123,
4913         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4914     }, {
4915         'note': 'even larger playlist, 8832 videos',
4916         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4917         'only_matching': True,
4918     }, {
4919         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4920         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4921         'info_dict': {
4922             'title': 'Uploads from Interstellar Movie',
4923             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4924             'uploader': 'Interstellar Movie',
4925             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4926             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4927             'tags': [],
4928             'view_count': int,
4929             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4930             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4931             'channel': 'Interstellar Movie',
4932             'description': '',
4933             'modified_date': r're:\d{8}',
4934         },
4935         'playlist_mincount': 21,
4936     }, {
4937         'note': 'Playlist with "show unavailable videos" button',
4938         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4939         'info_dict': {
4940             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4941             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4942             'uploader': 'Phim Siêu Nhân Nhật Bản',
4943             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4944             'view_count': int,
4945             'channel': 'Phim Siêu Nhân Nhật Bản',
4946             'tags': [],
4947             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4948             'description': '',
4949             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4950             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4951             'modified_date': r're:\d{8}',
4952         },
4953         'playlist_mincount': 200,
4954         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4955     }, {
4956         'note': 'Playlist with unavailable videos in page 7',
4957         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4958         'info_dict': {
4959             'title': 'Uploads from BlankTV',
4960             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4961             'uploader': 'BlankTV',
4962             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4963             'channel': 'BlankTV',
4964             'channel_url': 'https://www.youtube.com/c/blanktv',
4965             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4966             'view_count': int,
4967             'tags': [],
4968             'uploader_url': 'https://www.youtube.com/c/blanktv',
4969             'modified_date': r're:\d{8}',
4970             'description': '',
4971         },
4972         'playlist_mincount': 1000,
4973         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4974     }, {
4975         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4976         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4977         'info_dict': {
4978             'title': 'Data Analysis with Dr Mike Pound',
4979             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4980             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4981             'uploader': 'Computerphile',
4982             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4983             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4984             'tags': [],
4985             'view_count': int,
4986             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4987             'channel_url': 'https://www.youtube.com/user/Computerphile',
4988             'channel': 'Computerphile',
4989         },
4990         'playlist_mincount': 11,
4991     }, {
4992         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4993         'only_matching': True,
4994     }, {
4995         'note': 'Playlist URL that does not actually serve a playlist',
4996         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4997         'info_dict': {
4998             'id': 'FqZTN594JQw',
4999             'ext': 'webm',
5000             'title': "Smiley's People 01 detective, Adventure Series, Action",
5001             'uploader': 'STREEM',
5002             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5003             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5004             'upload_date': '20150526',
5005             'license': 'Standard YouTube License',
5006             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5007             'categories': ['People & Blogs'],
5008             'tags': list,
5009             'view_count': int,
5010             'like_count': int,
5011         },
5012         'params': {
5013             'skip_download': True,
5014         },
5015         'skip': 'This video is not available.',
5016         'add_ie': [YoutubeIE.ie_key()],
5017     }, {
5018         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5019         'only_matching': True,
5020     }, {
5021         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5022         'only_matching': True,
5023     }, {
5024         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5025         'info_dict': {
5026             'id': 'Wq15eF5vCbI',  # This will keep changing
5027             'ext': 'mp4',
5028             'title': str,
5029             'uploader': 'Sky News',
5030             'uploader_id': 'skynews',
5031             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5032             'upload_date': r're:\d{8}',
5033             'description': str,
5034             'categories': ['News & Politics'],
5035             'tags': list,
5036             'like_count': int,
5037             'release_timestamp': 1642502819,
5038             'channel': 'Sky News',
5039             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5040             'age_limit': 0,
5041             'view_count': int,
5042             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5043             'playable_in_embed': True,
5044             'release_date': '20220118',
5045             'availability': 'public',
5046             'live_status': 'is_live',
5047             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5048             'channel_follower_count': int
5049         },
5050         'params': {
5051             'skip_download': True,
5052         },
5053         'expected_warnings': ['Ignoring subtitle tracks found in '],
5054     }, {
5055         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5056         'info_dict': {
5057             'id': 'a48o2S1cPoo',
5058             'ext': 'mp4',
5059             'title': 'The Young Turks - Live Main Show',
5060             'uploader': 'The Young Turks',
5061             'uploader_id': 'TheYoungTurks',
5062             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5063             'upload_date': '20150715',
5064             'license': 'Standard YouTube License',
5065             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5066             'categories': ['News & Politics'],
5067             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5068             'like_count': int,
5069         },
5070         'params': {
5071             'skip_download': True,
5072         },
5073         'only_matching': True,
5074     }, {
5075         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5076         'only_matching': True,
5077     }, {
5078         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5079         'only_matching': True,
5080     }, {
5081         'note': 'A channel that is not live. Should raise error',
5082         'url': 'https://www.youtube.com/user/numberphile/live',
5083         'only_matching': True,
5084     }, {
5085         'url': 'https://www.youtube.com/feed/trending',
5086         'only_matching': True,
5087     }, {
5088         'url': 'https://www.youtube.com/feed/library',
5089         'only_matching': True,
5090     }, {
5091         'url': 'https://www.youtube.com/feed/history',
5092         'only_matching': True,
5093     }, {
5094         'url': 'https://www.youtube.com/feed/subscriptions',
5095         'only_matching': True,
5096     }, {
5097         'url': 'https://www.youtube.com/feed/watch_later',
5098         'only_matching': True,
5099     }, {
5100         'note': 'Recommended - redirects to home page.',
5101         'url': 'https://www.youtube.com/feed/recommended',
5102         'only_matching': True,
5103     }, {
5104         'note': 'inline playlist with not always working continuations',
5105         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5106         'only_matching': True,
5107     }, {
5108         'url': 'https://www.youtube.com/course',
5109         'only_matching': True,
5110     }, {
5111         'url': 'https://www.youtube.com/zsecurity',
5112         'only_matching': True,
5113     }, {
5114         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5115         'only_matching': True,
5116     }, {
5117         'url': 'https://www.youtube.com/TheYoungTurks/live',
5118         'only_matching': True,
5119     }, {
5120         'url': 'https://www.youtube.com/hashtag/cctv9',
5121         'info_dict': {
5122             'id': 'cctv9',
5123             'title': '#cctv9',
5124             'tags': [],
5125         },
5126         'playlist_mincount': 350,
5127     }, {
5128         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5129         'only_matching': True,
5130     }, {
5131         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5132         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5133         'only_matching': True
5134     }, {
5135         'note': '/browse/ should redirect to /channel/',
5136         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5137         'only_matching': True
5138     }, {
5139         'note': 'VLPL, should redirect to playlist?list=PL...',
5140         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5141         'info_dict': {
5142             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5143             'uploader': 'NoCopyrightSounds',
5144             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5145             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5146             'title': 'NCS : All Releases 💿',
5147             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5148             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5149             'modified_date': r're:\d{8}',
5150             'view_count': int,
5151             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5152             'tags': [],
5153             'channel': 'NoCopyrightSounds',
5154         },
5155         'playlist_mincount': 166,
5156         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5157     }, {
5158         'note': 'Topic, should redirect to playlist?list=UU...',
5159         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5160         'info_dict': {
5161             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5162             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5163             'title': 'Uploads from Royalty Free Music - Topic',
5164             'uploader': 'Royalty Free Music - Topic',
5165             'tags': [],
5166             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5167             'channel': 'Royalty Free Music - Topic',
5168             'view_count': int,
5169             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5170             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5171             'modified_date': r're:\d{8}',
5172             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5173             'description': '',
5174         },
5175         'expected_warnings': [
5176             'The URL does not have a videos tab',
5177             r'[Uu]navailable videos (are|will be) hidden',
5178         ],
5179         'playlist_mincount': 101,
5180     }, {
5181         'note': 'Topic without a UU playlist',
5182         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5183         'info_dict': {
5184             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5185             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5186             'tags': [],
5187         },
5188         'expected_warnings': [
5189             'the playlist redirect gave error',
5190         ],
5191         'playlist_mincount': 9,
5192     }, {
5193         'note': 'Youtube music Album',
5194         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5195         'info_dict': {
5196             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5197             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5198             'tags': [],
5199             'view_count': int,
5200             'description': '',
5201             'availability': 'unlisted',
5202             'modified_date': r're:\d{8}',
5203         },
5204         'playlist_count': 50,
5205     }, {
5206         'note': 'unlisted single video playlist',
5207         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5208         'info_dict': {
5209             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5210             'uploader': 'colethedj',
5211             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5212             'title': 'yt-dlp unlisted playlist test',
5213             'availability': 'unlisted',
5214             'tags': [],
5215             'modified_date': '20220418',
5216             'channel': 'colethedj',
5217             'view_count': int,
5218             'description': '',
5219             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5220             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5221             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5222         },
5223         'playlist_count': 1,
5224     }, {
5225         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5226         'url': 'https://www.youtube.com/feed/recommended',
5227         'info_dict': {
5228             'id': 'recommended',
5229             'title': 'recommended',
5230             'tags': [],
5231         },
5232         'playlist_mincount': 50,
5233         'params': {
5234             'skip_download': True,
5235             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5236         },
5237     }, {
5238         'note': 'API Fallback: /videos tab, sorted by oldest first',
5239         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5240         'info_dict': {
5241             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5242             'title': 'Cody\'sLab - Videos',
5243             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5244             'uploader': 'Cody\'sLab',
5245             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5246             'channel': 'Cody\'sLab',
5247             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5248             'tags': [],
5249             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5250             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5251             'channel_follower_count': int
5252         },
5253         'playlist_mincount': 650,
5254         'params': {
5255             'skip_download': True,
5256             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5257         },
5258     }, {
5259         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5260         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5261         'info_dict': {
5262             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5263             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5264             'title': 'Uploads from Royalty Free Music - Topic',
5265             'uploader': 'Royalty Free Music - Topic',
5266             'modified_date': r're:\d{8}',
5267             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5268             'description': '',
5269             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5270             'tags': [],
5271             'channel': 'Royalty Free Music - Topic',
5272             'view_count': int,
5273             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5274         },
5275         'expected_warnings': [
5276             'does not have a videos tab',
5277             r'[Uu]navailable videos (are|will be) hidden',
5278         ],
5279         'playlist_mincount': 101,
5280         'params': {
5281             'skip_download': True,
5282             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5283         },
5284     }, {
5285         'note': 'non-standard redirect to regional channel',
5286         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5287         'only_matching': True
5288     }, {
5289         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5290         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5291         'info_dict': {
5292             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5293             'modified_date': '20220407',
5294             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5295             'tags': [],
5296             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5297             'uploader': 'pukkandan',
5298             'availability': 'unlisted',
5299             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5300             'channel': 'pukkandan',
5301             'description': 'Test for collaborative playlist',
5302             'title': 'yt-dlp test - collaborative playlist',
5303             'view_count': int,
5304             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5305         },
5306         'playlist_mincount': 2
5307     }]
5308
5309     @classmethod
5310     def suitable(cls, url):
5311         return False if YoutubeIE.suitable(url) else super().suitable(url)
5312
5313     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5314
5315     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5316     def _real_extract(self, url, smuggled_data):
5317         item_id = self._match_id(url)
5318         url = urllib.parse.urlunparse(
5319             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5320         compat_opts = self.get_param('compat_opts', [])
5321
5322         def get_mobj(url):
5323             mobj = self._URL_RE.match(url).groupdict()
5324             mobj.update((k, '') for k, v in mobj.items() if v is None)
5325             return mobj
5326
5327         mobj, redirect_warning = get_mobj(url), None
5328         # Youtube returns incomplete data if tabname is not lower case
5329         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5330         if is_channel:
5331             if smuggled_data.get('is_music_url'):
5332                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5333                     item_id = item_id[2:]
5334                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5335                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5336                     mdata = self._extract_tab_endpoint(
5337                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5338                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5339                                         get_all=False, expected_type=str)
5340                     if not murl:
5341                         raise ExtractorError('Failed to resolve album to playlist')
5342                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5343                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5344                     pre = f'https://www.youtube.com/channel/{item_id}'
5345
5346         original_tab_name = tab
5347         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5348             # Home URLs should redirect to /videos/
5349             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5350                                 'To download only the videos in the home page, add a "/featured" to the URL')
5351             tab = '/videos'
5352
5353         url = ''.join((pre, tab, post))
5354         mobj = get_mobj(url)
5355
5356         # Handle both video/playlist URLs
5357         qs = parse_qs(url)
5358         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5359
5360         if not video_id and mobj['not_channel'].startswith('watch'):
5361             if not playlist_id:
5362                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5363                 raise ExtractorError('Unable to recognize tab page')
5364             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5365             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5366             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5367             mobj = get_mobj(url)
5368
5369         if video_id and playlist_id:
5370             if self.get_param('noplaylist'):
5371                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5372                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5373                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5374             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5375
5376         data, ytcfg = self._extract_data(url, item_id)
5377
5378         # YouTube may provide a non-standard redirect to the regional channel
5379         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5380         redirect_url = traverse_obj(
5381             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5382         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5383             redirect_url = ''.join((
5384                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5385             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5386             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5387
5388         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5389         if tabs:
5390             selected_tab = self._extract_selected_tab(tabs)
5391             selected_tab_name = selected_tab.get('title', '').lower()
5392             if selected_tab_name == 'home':
5393                 selected_tab_name = 'featured'
5394             requested_tab_name = mobj['tab'][1:]
5395             if 'no-youtube-channel-redirect' not in compat_opts:
5396                 if requested_tab_name == 'live':  # Live tab should have redirected to the video
5397                     raise UserNotLive(video_id=mobj['id'])
5398                 if requested_tab_name not in ('', selected_tab_name):
5399                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5400                     if not original_tab_name:
5401                         if item_id[:2] == 'UC':
5402                             # Topic channels don't have /videos. Use the equivalent playlist instead
5403                             pl_id = f'UU{item_id[2:]}'
5404                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5405                             try:
5406                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5407                             except ExtractorError:
5408                                 redirect_warning += ' and the playlist redirect gave error'
5409                             else:
5410                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5411                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5412                         if selected_tab_name and selected_tab_name != requested_tab_name:
5413                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5414                     else:
5415                         raise ExtractorError(redirect_warning, expected=True)
5416
5417         if redirect_warning:
5418             self.to_screen(redirect_warning)
5419         self.write_debug(f'Final URL: {url}')
5420
5421         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5422         if 'no-youtube-unavailable-videos' not in compat_opts:
5423             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5424         self._extract_and_report_alerts(data, only_once=True)
5425         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5426         if tabs:
5427             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5428
5429         playlist = traverse_obj(
5430             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5431         if playlist:
5432             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5433
5434         video_id = traverse_obj(
5435             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5436         if video_id:
5437             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5438                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5439             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5440                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5441
5442         raise ExtractorError('Unable to recognize tab page')
5443
5444
5445 class YoutubePlaylistIE(InfoExtractor):
5446     IE_DESC = 'YouTube playlists'
5447     _VALID_URL = r'''(?x)(?:
5448                         (?:https?://)?
5449                         (?:\w+\.)?
5450                         (?:
5451                             (?:
5452                                 youtube(?:kids)?\.com|
5453                                 %(invidious)s
5454                             )
5455                             /.*?\?.*?\blist=
5456                         )?
5457                         (?P<id>%(playlist_id)s)
5458                      )''' % {
5459         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5460         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5461     }
5462     IE_NAME = 'youtube:playlist'
5463     _TESTS = [{
5464         'note': 'issue #673',
5465         'url': 'PLBB231211A4F62143',
5466         'info_dict': {
5467             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5468             'id': 'PLBB231211A4F62143',
5469             'uploader': 'Wickman',
5470             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5471             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5472             'view_count': int,
5473             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5474             'modified_date': r're:\d{8}',
5475             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5476             'channel': 'Wickman',
5477             'tags': [],
5478             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5479         },
5480         'playlist_mincount': 29,
5481     }, {
5482         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5483         'info_dict': {
5484             'title': 'YDL_safe_search',
5485             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5486         },
5487         'playlist_count': 2,
5488         'skip': 'This playlist is private',
5489     }, {
5490         'note': 'embedded',
5491         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5492         'playlist_count': 4,
5493         'info_dict': {
5494             'title': 'JODA15',
5495             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5496             'uploader': 'milan',
5497             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5498             'description': '',
5499             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5500             'tags': [],
5501             'modified_date': '20140919',
5502             'view_count': int,
5503             'channel': 'milan',
5504             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5505             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5506         },
5507         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5508     }, {
5509         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5510         'playlist_mincount': 455,
5511         'info_dict': {
5512             'title': '2018 Chinese New Singles (11/6 updated)',
5513             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5514             'uploader': 'LBK',
5515             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5516             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5517             'channel': 'LBK',
5518             'view_count': int,
5519             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5520             'tags': [],
5521             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5522             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5523             'modified_date': r're:\d{8}',
5524         },
5525         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5526     }, {
5527         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5528         'only_matching': True,
5529     }, {
5530         # music album playlist
5531         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5532         'only_matching': True,
5533     }]
5534
5535     @classmethod
5536     def suitable(cls, url):
5537         if YoutubeTabIE.suitable(url):
5538             return False
5539         from ..utils import parse_qs
5540         qs = parse_qs(url)
5541         if qs.get('v', [None])[0]:
5542             return False
5543         return super().suitable(url)
5544
5545     def _real_extract(self, url):
5546         playlist_id = self._match_id(url)
5547         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5548         url = update_url_query(
5549             'https://www.youtube.com/playlist',
5550             parse_qs(url) or {'list': playlist_id})
5551         if is_music_url:
5552             url = smuggle_url(url, {'is_music_url': True})
5553         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5554
5555
5556 class YoutubeYtBeIE(InfoExtractor):
5557     IE_DESC = 'youtu.be'
5558     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5559     _TESTS = [{
5560         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5561         'info_dict': {
5562             'id': 'yeWKywCrFtk',
5563             'ext': 'mp4',
5564             'title': 'Small Scale Baler and Braiding Rugs',
5565             'uploader': 'Backus-Page House Museum',
5566             'uploader_id': 'backuspagemuseum',
5567             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5568             'upload_date': '20161008',
5569             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5570             'categories': ['Nonprofits & Activism'],
5571             'tags': list,
5572             'like_count': int,
5573             'age_limit': 0,
5574             'playable_in_embed': True,
5575             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5576             'channel': 'Backus-Page House Museum',
5577             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5578             'live_status': 'not_live',
5579             'view_count': int,
5580             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5581             'availability': 'public',
5582             'duration': 59,
5583             'comment_count': int,
5584             'channel_follower_count': int
5585         },
5586         'params': {
5587             'noplaylist': True,
5588             'skip_download': True,
5589         },
5590     }, {
5591         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5592         'only_matching': True,
5593     }]
5594
5595     def _real_extract(self, url):
5596         mobj = self._match_valid_url(url)
5597         video_id = mobj.group('id')
5598         playlist_id = mobj.group('playlist_id')
5599         return self.url_result(
5600             update_url_query('https://www.youtube.com/watch', {
5601                 'v': video_id,
5602                 'list': playlist_id,
5603                 'feature': 'youtu.be',
5604             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5605
5606
5607 class YoutubeLivestreamEmbedIE(InfoExtractor):
5608     IE_DESC = 'YouTube livestream embeds'
5609     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5610     _TESTS = [{
5611         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5612         'only_matching': True,
5613     }]
5614
5615     def _real_extract(self, url):
5616         channel_id = self._match_id(url)
5617         return self.url_result(
5618             f'https://www.youtube.com/channel/{channel_id}/live',
5619             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5620
5621
5622 class YoutubeYtUserIE(InfoExtractor):
5623     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5624     IE_NAME = 'youtube:user'
5625     _VALID_URL = r'ytuser:(?P<id>.+)'
5626     _TESTS = [{
5627         'url': 'ytuser:phihag',
5628         'only_matching': True,
5629     }]
5630
5631     def _real_extract(self, url):
5632         user_id = self._match_id(url)
5633         return self.url_result(
5634             'https://www.youtube.com/user/%s/videos' % user_id,
5635             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5636
5637
5638 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5639     IE_NAME = 'youtube:favorites'
5640     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5641     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5642     _LOGIN_REQUIRED = True
5643     _TESTS = [{
5644         'url': ':ytfav',
5645         'only_matching': True,
5646     }, {
5647         'url': ':ytfavorites',
5648         'only_matching': True,
5649     }]
5650
5651     def _real_extract(self, url):
5652         return self.url_result(
5653             'https://www.youtube.com/playlist?list=LL',
5654             ie=YoutubeTabIE.ie_key())
5655
5656
5657 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5658     IE_NAME = 'youtube:notif'
5659     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5660     _VALID_URL = r':ytnotif(?:ication)?s?'
5661     _LOGIN_REQUIRED = True
5662     _TESTS = [{
5663         'url': ':ytnotif',
5664         'only_matching': True,
5665     }, {
5666         'url': ':ytnotifications',
5667         'only_matching': True,
5668     }]
5669
5670     def _extract_notification_menu(self, response, continuation_list):
5671         notification_list = traverse_obj(
5672             response,
5673             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5674             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5675             expected_type=list) or []
5676         continuation_list[0] = None
5677         for item in notification_list:
5678             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5679             if entry:
5680                 yield entry
5681             continuation = item.get('continuationItemRenderer')
5682             if continuation:
5683                 continuation_list[0] = continuation
5684
5685     def _extract_notification_renderer(self, notification):
5686         video_id = traverse_obj(
5687             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5688         url = f'https://www.youtube.com/watch?v={video_id}'
5689         channel_id = None
5690         if not video_id:
5691             browse_ep = traverse_obj(
5692                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5693             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5694             post_id = self._search_regex(
5695                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5696                 'post id', default=None)
5697             if not channel_id or not post_id:
5698                 return
5699             # The direct /post url redirects to this in the browser
5700             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5701
5702         channel = traverse_obj(
5703             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5704             expected_type=str)
5705         notification_title = self._get_text(notification, 'shortMessage')
5706         if notification_title:
5707             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
5708         # TODO: handle recommended videos
5709         title = self._search_regex(
5710             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
5711             'video title', default=None)
5712         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5713                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5714                        else None)
5715         return {
5716             '_type': 'url',
5717             'url': url,
5718             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5719             'video_id': video_id,
5720             'title': title,
5721             'channel_id': channel_id,
5722             'channel': channel,
5723             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5724             'upload_date': upload_date,
5725         }
5726
5727     def _notification_menu_entries(self, ytcfg):
5728         continuation_list = [None]
5729         response = None
5730         for page in itertools.count(1):
5731             ctoken = traverse_obj(
5732                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5733             response = self._extract_response(
5734                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5735                 ep='notification/get_notification_menu', check_get_keys='actions',
5736                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5737             yield from self._extract_notification_menu(response, continuation_list)
5738             if not continuation_list[0]:
5739                 break
5740
5741     def _real_extract(self, url):
5742         display_id = 'notifications'
5743         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5744         self._report_playlist_authcheck(ytcfg)
5745         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5746
5747
5748 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5749     IE_DESC = 'YouTube search'
5750     IE_NAME = 'youtube:search'
5751     _SEARCH_KEY = 'ytsearch'
5752     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5753     _TESTS = [{
5754         'url': 'ytsearch5:youtube-dl test video',
5755         'playlist_count': 5,
5756         'info_dict': {
5757             'id': 'youtube-dl test video',
5758             'title': 'youtube-dl test video',
5759         }
5760     }]
5761
5762
5763 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5764     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5765     _SEARCH_KEY = 'ytsearchdate'
5766     IE_DESC = 'YouTube search, newest videos first'
5767     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5768     _TESTS = [{
5769         'url': 'ytsearchdate5:youtube-dl test video',
5770         'playlist_count': 5,
5771         'info_dict': {
5772             'id': 'youtube-dl test video',
5773             'title': 'youtube-dl test video',
5774         }
5775     }]
5776
5777
5778 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5779     IE_DESC = 'YouTube search URLs with sorting and filter support'
5780     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5781     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5782     _TESTS = [{
5783         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5784         'playlist_mincount': 5,
5785         'info_dict': {
5786             'id': 'youtube-dl test video',
5787             'title': 'youtube-dl test video',
5788         }
5789     }, {
5790         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5791         'playlist_mincount': 5,
5792         'info_dict': {
5793             'id': 'python',
5794             'title': 'python',
5795         }
5796     }, {
5797         'url': 'https://www.youtube.com/results?search_query=%23cats',
5798         'playlist_mincount': 1,
5799         'info_dict': {
5800             'id': '#cats',
5801             'title': '#cats',
5802             # The test suite does not have support for nested playlists
5803             # 'entries': [{
5804             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5805             #     'title': '#cats',
5806             # }],
5807         },
5808     }, {
5809         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5810         'only_matching': True,
5811     }]
5812
5813     def _real_extract(self, url):
5814         qs = parse_qs(url)
5815         query = (qs.get('search_query') or qs.get('q'))[0]
5816         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5817
5818
5819 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5820     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5821     IE_NAME = 'youtube:music:search_url'
5822     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5823     _TESTS = [{
5824         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5825         'playlist_count': 16,
5826         'info_dict': {
5827             'id': 'royalty free music',
5828             'title': 'royalty free music',
5829         }
5830     }, {
5831         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5832         'playlist_mincount': 30,
5833         'info_dict': {
5834             'id': 'royalty free music - songs',
5835             'title': 'royalty free music - songs',
5836         },
5837         'params': {'extract_flat': 'in_playlist'}
5838     }, {
5839         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5840         'playlist_mincount': 30,
5841         'info_dict': {
5842             'id': 'royalty free music - community playlists',
5843             'title': 'royalty free music - community playlists',
5844         },
5845         'params': {'extract_flat': 'in_playlist'}
5846     }]
5847
5848     _SECTIONS = {
5849         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5850         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5851         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5852         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5853         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5854         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5855     }
5856
5857     def _real_extract(self, url):
5858         qs = parse_qs(url)
5859         query = (qs.get('search_query') or qs.get('q'))[0]
5860         params = qs.get('sp', (None,))[0]
5861         if params:
5862             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5863         else:
5864             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
5865             params = self._SECTIONS.get(section)
5866             if not params:
5867                 section = None
5868         title = join_nonempty(query, section, delim=' - ')
5869         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5870
5871
5872 class YoutubeFeedsInfoExtractor(InfoExtractor):
5873     """
5874     Base class for feed extractors
5875     Subclasses must re-define the _FEED_NAME property.
5876     """
5877     _LOGIN_REQUIRED = True
5878     _FEED_NAME = 'feeds'
5879
5880     def _real_initialize(self):
5881         YoutubeBaseInfoExtractor._check_login_required(self)
5882
5883     @classproperty
5884     def IE_NAME(self):
5885         return f'youtube:{self._FEED_NAME}'
5886
5887     def _real_extract(self, url):
5888         return self.url_result(
5889             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5890
5891
5892 class YoutubeWatchLaterIE(InfoExtractor):
5893     IE_NAME = 'youtube:watchlater'
5894     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5895     _VALID_URL = r':ytwatchlater'
5896     _TESTS = [{
5897         'url': ':ytwatchlater',
5898         'only_matching': True,
5899     }]
5900
5901     def _real_extract(self, url):
5902         return self.url_result(
5903             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5904
5905
5906 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5907     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5908     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5909     _FEED_NAME = 'recommended'
5910     _LOGIN_REQUIRED = False
5911     _TESTS = [{
5912         'url': ':ytrec',
5913         'only_matching': True,
5914     }, {
5915         'url': ':ytrecommended',
5916         'only_matching': True,
5917     }, {
5918         'url': 'https://youtube.com',
5919         'only_matching': True,
5920     }]
5921
5922
5923 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5924     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5925     _VALID_URL = r':ytsub(?:scription)?s?'
5926     _FEED_NAME = 'subscriptions'
5927     _TESTS = [{
5928         'url': ':ytsubs',
5929         'only_matching': True,
5930     }, {
5931         'url': ':ytsubscriptions',
5932         'only_matching': True,
5933     }]
5934
5935
5936 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5937     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5938     _VALID_URL = r':ythis(?:tory)?'
5939     _FEED_NAME = 'history'
5940     _TESTS = [{
5941         'url': ':ythistory',
5942         'only_matching': True,
5943     }]
5944
5945
5946 class YoutubeStoriesIE(InfoExtractor):
5947     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5948     IE_NAME = 'youtube:stories'
5949     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5950     _TESTS = [{
5951         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5952         'only_matching': True,
5953     }]
5954
5955     def _real_extract(self, url):
5956         playlist_id = f'RLTD{self._match_id(url)}'
5957         return self.url_result(
5958             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5959             ie=YoutubeTabIE, video_id=playlist_id)
5960
5961
5962 class YoutubeTruncatedURLIE(InfoExtractor):
5963     IE_NAME = 'youtube:truncated_url'
5964     IE_DESC = False  # Do not list
5965     _VALID_URL = r'''(?x)
5966         (?:https?://)?
5967         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5968         (?:watch\?(?:
5969             feature=[a-z_]+|
5970             annotation_id=annotation_[^&]+|
5971             x-yt-cl=[0-9]+|
5972             hl=[^&]*|
5973             t=[0-9]+
5974         )?
5975         |
5976             attribution_link\?a=[^&]+
5977         )
5978         $
5979     '''
5980
5981     _TESTS = [{
5982         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5983         'only_matching': True,
5984     }, {
5985         'url': 'https://www.youtube.com/watch?',
5986         'only_matching': True,
5987     }, {
5988         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5989         'only_matching': True,
5990     }, {
5991         'url': 'https://www.youtube.com/watch?feature=foo',
5992         'only_matching': True,
5993     }, {
5994         'url': 'https://www.youtube.com/watch?hl=en-GB',
5995         'only_matching': True,
5996     }, {
5997         'url': 'https://www.youtube.com/watch?t=2372',
5998         'only_matching': True,
5999     }]
6000
6001     def _real_extract(self, url):
6002         raise ExtractorError(
6003             'Did you forget to quote the URL? Remember that & is a meta '
6004             'character in most shells, so you want to put the URL in quotes, '
6005             'like  youtube-dl '
6006             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6007             ' or simply  youtube-dl BaW_jenozKc  .',
6008             expected=True)
6009
6010
6011 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6012     IE_NAME = 'youtube:clip'
6013     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6014     _TESTS = [{
6015         # FIXME: Other metadata should be extracted from the clip, not from the base video
6016         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6017         'info_dict': {
6018             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6019             'ext': 'mp4',
6020             'section_start': 29.0,
6021             'section_end': 39.7,
6022             'duration': 10.7,
6023             'age_limit': 0,
6024             'availability': 'public',
6025             'categories': ['Gaming'],
6026             'channel': 'Scott The Woz',
6027             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6028             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6029             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6030             'like_count': int,
6031             'playable_in_embed': True,
6032             'tags': 'count:17',
6033             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6034             'title': 'Mobile Games on Console - Scott The Woz',
6035             'upload_date': '20210920',
6036             'uploader': 'Scott The Woz',
6037             'uploader_id': 'scottthewoz',
6038             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6039             'view_count': int,
6040             'live_status': 'not_live',
6041             'channel_follower_count': int
6042         }
6043     }]
6044
6045     def _real_extract(self, url):
6046         clip_id = self._match_id(url)
6047         _, data = self._extract_webpage(url, clip_id)
6048
6049         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6050         if not video_id:
6051             raise ExtractorError('Unable to find video ID')
6052
6053         clip_data = traverse_obj(data, (
6054             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6055             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6056             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6057             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6058
6059         return {
6060             '_type': 'url_transparent',
6061             'url': f'https://www.youtube.com/watch?v={video_id}',
6062             'ie_key': YoutubeIE.ie_key(),
6063             'id': clip_id,
6064             'section_start': int(clip_data['startTimeMs']) / 1000,
6065             'section_end': int(clip_data['endTimeMs']) / 1000,
6066         }
6067
6068
6069 class YoutubeTruncatedIDIE(InfoExtractor):
6070     IE_NAME = 'youtube:truncated_id'
6071     IE_DESC = False  # Do not list
6072     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6073
6074     _TESTS = [{
6075         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6076         'only_matching': True,
6077     }]
6078
6079     def _real_extract(self, url):
6080         video_id = self._match_id(url)
6081         raise ExtractorError(
6082             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6083             expected=True)