yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16 import urllib.parse
  17
  18 from .common import InfoExtractor, SearchInfoExtractor
  19 from ..compat import functools  # isort: split
  20 from ..compat import (
  21     compat_HTTPError,
  22     compat_parse_qs,
  23     compat_str,
  24     compat_urllib_parse_urlencode,
  25     compat_urllib_parse_urlparse,
  26     compat_urlparse,
  27 )
  28 from ..jsinterp import JSInterpreter
  29 from ..utils import (
  30     NO_DEFAULT,
  31     ExtractorError,
  32     bug_reports_message,
  33     classproperty,
  34     clean_html,
  35     datetime_from_str,
  36     dict_get,
  37     error_to_compat_str,
  38     float_or_none,
  39     format_field,
  40     get_first,
  41     int_or_none,
  42     is_html,
  43     join_nonempty,
  44     js_to_json,
  45     mimetype2ext,
  46     network_exceptions,
  47     orderedSet,
  48     parse_codecs,
  49     parse_count,
  50     parse_duration,
  51     parse_iso8601,
  52     parse_qs,
  53     qualities,
  54     remove_end,
  55     remove_start,
  56     smuggle_url,
  57     str_or_none,
  58     str_to_int,
  59     strftime_or_none,
  60     traverse_obj,
  61     try_get,
  62     unescapeHTML,
  63     unified_strdate,
  64     unified_timestamp,
  65     unsmuggle_url,
  66     update_url_query,
  67     url_or_none,
  68     urljoin,
  69     variadic,
  70 )
  71
  72 # any clients starting with _ cannot be explicity requested by the user
  73 INNERTUBE_CLIENTS = {
  74     'web': {
  75         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  76         'INNERTUBE_CONTEXT': {
  77             'client': {
  78                 'clientName': 'WEB',
  79                 'clientVersion': '2.20211221.00.00',
  80             }
  81         },
  82         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  83     },
  84     'web_embedded': {
  85         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  86         'INNERTUBE_CONTEXT': {
  87             'client': {
  88                 'clientName': 'WEB_EMBEDDED_PLAYER',
  89                 'clientVersion': '1.20211215.00.01',
  90             },
  91         },
  92         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  93     },
  94     'web_music': {
  95         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  96         'INNERTUBE_HOST': 'music.youtube.com',
  97         'INNERTUBE_CONTEXT': {
  98             'client': {
  99                 'clientName': 'WEB_REMIX',
 100                 'clientVersion': '1.20211213.00.00',
 101             }
 102         },
 103         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 104     },
 105     'web_creator': {
 106         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 107         'INNERTUBE_CONTEXT': {
 108             'client': {
 109                 'clientName': 'WEB_CREATOR',
 110                 'clientVersion': '1.20211220.02.00',
 111             }
 112         },
 113         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 114     },
 115     'android': {
 116         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 117         'INNERTUBE_CONTEXT': {
 118             'client': {
 119                 'clientName': 'ANDROID',
 120                 'clientVersion': '16.49',
 121             }
 122         },
 123         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 124         'REQUIRE_JS_PLAYER': False
 125     },
 126     'android_embedded': {
 127         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 128         'INNERTUBE_CONTEXT': {
 129             'client': {
 130                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 131                 'clientVersion': '16.49',
 132             },
 133         },
 134         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 135         'REQUIRE_JS_PLAYER': False
 136     },
 137     'android_music': {
 138         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 139         'INNERTUBE_CONTEXT': {
 140             'client': {
 141                 'clientName': 'ANDROID_MUSIC',
 142                 'clientVersion': '4.57',
 143             }
 144         },
 145         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 146         'REQUIRE_JS_PLAYER': False
 147     },
 148     'android_creator': {
 149         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 150         'INNERTUBE_CONTEXT': {
 151             'client': {
 152                 'clientName': 'ANDROID_CREATOR',
 153                 'clientVersion': '21.47',
 154             },
 155         },
 156         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 157         'REQUIRE_JS_PLAYER': False
 158     },
 159     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 160     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 161     'ios': {
 162         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 163         'INNERTUBE_CONTEXT': {
 164             'client': {
 165                 'clientName': 'IOS',
 166                 'clientVersion': '16.46',
 167                 'deviceModel': 'iPhone14,3',
 168             }
 169         },
 170         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 171         'REQUIRE_JS_PLAYER': False
 172     },
 173     'ios_embedded': {
 174         'INNERTUBE_CONTEXT': {
 175             'client': {
 176                 'clientName': 'IOS_MESSAGES_EXTENSION',
 177                 'clientVersion': '16.46',
 178                 'deviceModel': 'iPhone14,3',
 179             },
 180         },
 181         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 182         'REQUIRE_JS_PLAYER': False
 183     },
 184     'ios_music': {
 185         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 186         'INNERTUBE_CONTEXT': {
 187             'client': {
 188                 'clientName': 'IOS_MUSIC',
 189                 'clientVersion': '4.57',
 190             },
 191         },
 192         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 193         'REQUIRE_JS_PLAYER': False
 194     },
 195     'ios_creator': {
 196         'INNERTUBE_CONTEXT': {
 197             'client': {
 198                 'clientName': 'IOS_CREATOR',
 199                 'clientVersion': '21.47',
 200             },
 201         },
 202         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 203         'REQUIRE_JS_PLAYER': False
 204     },
 205     # mweb has 'ultralow' formats
 206     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 207     'mweb': {
 208         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 209         'INNERTUBE_CONTEXT': {
 210             'client': {
 211                 'clientName': 'MWEB',
 212                 'clientVersion': '2.20211221.01.00',
 213             }
 214         },
 215         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 216     },
 217     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 218     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 219     'tv_embedded': {
 220         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 221         'INNERTUBE_CONTEXT': {
 222             'client': {
 223                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 224                 'clientVersion': '2.0',
 225             },
 226         },
 227         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 228     },
 229 }
 230
 231
 232 def _split_innertube_client(client_name):
 233     variant, *base = client_name.rsplit('.', 1)
 234     if base:
 235         return variant, base[0], variant
 236     base, *variant = client_name.split('_', 1)
 237     return client_name, base, variant[0] if variant else None
 238
 239
 240 def build_innertube_clients():
 241     THIRD_PARTY = {
 242         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 243     }
 244     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 245     priority = qualities(BASE_CLIENTS[::-1])
 246
 247     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 248         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 249         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 250         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 251         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 252
 253         _, base_client, variant = _split_innertube_client(client)
 254         ytcfg['priority'] = 10 * priority(base_client)
 255
 256         if not variant:
 257             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 258             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 259             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 260             embedscreen['priority'] -= 3
 261         elif variant == 'embedded':
 262             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 263             ytcfg['priority'] -= 2
 264         else:
 265             ytcfg['priority'] -= 3
 266
 267
 268 build_innertube_clients()
 269
 270
 271 class YoutubeBaseInfoExtractor(InfoExtractor):
 272     """Provide base functions for Youtube extractors"""
 273
 274     _RESERVED_NAMES = (
 275         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 276         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 277         r'browse|oembed|get_video_info|iframe_api|s/player|'
 278         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 279
 280     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 281
 282     # _NETRC_MACHINE = 'youtube'
 283
 284     # If True it will raise an error if no login info is provided
 285     _LOGIN_REQUIRED = False
 286
 287     _INVIDIOUS_SITES = (
 288         # invidious-redirect websites
 289         r'(?:www\.)?redirect\.invidious\.io',
 290         r'(?:(?:www|dev)\.)?invidio\.us',
 291         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 292         r'(?:www\.)?invidious\.pussthecat\.org',
 293         r'(?:www\.)?invidious\.zee\.li',
 294         r'(?:www\.)?invidious\.ethibox\.fr',
 295         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 296         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 297         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 298         # youtube-dl invidious instances list
 299         r'(?:(?:www|no)\.)?invidiou\.sh',
 300         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 301         r'(?:www\.)?invidious\.kabi\.tk',
 302         r'(?:www\.)?invidious\.mastodon\.host',
 303         r'(?:www\.)?invidious\.zapashcanon\.fr',
 304         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 305         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 306         r'(?:www\.)?invidious\.himiko\.cloud',
 307         r'(?:www\.)?invidious\.reallyancient\.tech',
 308         r'(?:www\.)?invidious\.tube',
 309         r'(?:www\.)?invidiou\.site',
 310         r'(?:www\.)?invidious\.site',
 311         r'(?:www\.)?invidious\.xyz',
 312         r'(?:www\.)?invidious\.nixnet\.xyz',
 313         r'(?:www\.)?invidious\.048596\.xyz',
 314         r'(?:www\.)?invidious\.drycat\.fr',
 315         r'(?:www\.)?inv\.skyn3t\.in',
 316         r'(?:www\.)?tube\.poal\.co',
 317         r'(?:www\.)?tube\.connect\.cafe',
 318         r'(?:www\.)?vid\.wxzm\.sx',
 319         r'(?:www\.)?vid\.mint\.lgbt',
 320         r'(?:www\.)?vid\.puffyan\.us',
 321         r'(?:www\.)?yewtu\.be',
 322         r'(?:www\.)?yt\.elukerio\.org',
 323         r'(?:www\.)?yt\.lelux\.fi',
 324         r'(?:www\.)?invidious\.ggc-project\.de',
 325         r'(?:www\.)?yt\.maisputain\.ovh',
 326         r'(?:www\.)?ytprivate\.com',
 327         r'(?:www\.)?invidious\.13ad\.de',
 328         r'(?:www\.)?invidious\.toot\.koeln',
 329         r'(?:www\.)?invidious\.fdn\.fr',
 330         r'(?:www\.)?watch\.nettohikari\.com',
 331         r'(?:www\.)?invidious\.namazso\.eu',
 332         r'(?:www\.)?invidious\.silkky\.cloud',
 333         r'(?:www\.)?invidious\.exonip\.de',
 334         r'(?:www\.)?invidious\.riverside\.rocks',
 335         r'(?:www\.)?invidious\.blamefran\.net',
 336         r'(?:www\.)?invidious\.moomoo\.de',
 337         r'(?:www\.)?ytb\.trom\.tf',
 338         r'(?:www\.)?yt\.cyberhost\.uk',
 339         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 340         r'(?:www\.)?qklhadlycap4cnod\.onion',
 341         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 342         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 343         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 344         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 345         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 346         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 347         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 348         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 349         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 350         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 351         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 352         r'(?:www\.)?piped\.kavin\.rocks',
 353         r'(?:www\.)?piped\.silkky\.cloud',
 354         r'(?:www\.)?piped\.tokhmi\.xyz',
 355         r'(?:www\.)?piped\.moomoo\.me',
 356         r'(?:www\.)?il\.ax',
 357         r'(?:www\.)?piped\.syncpundit\.com',
 358         r'(?:www\.)?piped\.mha\.fi',
 359         r'(?:www\.)?piped\.mint\.lgbt',
 360         r'(?:www\.)?piped\.privacy\.com\.de',
 361     )
 362
 363     def _initialize_consent(self):
 364         cookies = self._get_cookies('https://www.youtube.com/')
 365         if cookies.get('__Secure-3PSID'):
 366             return
 367         consent_id = None
 368         consent = cookies.get('CONSENT')
 369         if consent:
 370             if 'YES' in consent.value:
 371                 return
 372             consent_id = self._search_regex(
 373                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 374         if not consent_id:
 375             consent_id = random.randint(100, 999)
 376         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 377
 378     def _initialize_pref(self):
 379         cookies = self._get_cookies('https://www.youtube.com/')
 380         pref_cookie = cookies.get('PREF')
 381         pref = {}
 382         if pref_cookie:
 383             try:
 384                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 385             except ValueError:
 386                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 387         pref.update({'hl': 'en', 'tz': 'UTC'})
 388         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 389
 390     def _real_initialize(self):
 391         self._initialize_pref()
 392         self._initialize_consent()
 393         self._check_login_required()
 394
 395     def _check_login_required(self):
 396         if self._LOGIN_REQUIRED and not self._cookies_passed:
 397             self.raise_login_required('Login details are needed to download this content', method='cookies')
 398
 399     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 400     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 401
 402     def _get_default_ytcfg(self, client='web'):
 403         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 404
 405     def _get_innertube_host(self, client='web'):
 406         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 407
 408     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 409         # try_get but with fallback to default ytcfg client values when present
 410         _func = lambda y: try_get(y, getter, expected_type)
 411         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 412
 413     def _extract_client_name(self, ytcfg, default_client='web'):
 414         return self._ytcfg_get_safe(
 415             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 416                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 417
 418     def _extract_client_version(self, ytcfg, default_client='web'):
 419         return self._ytcfg_get_safe(
 420             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 421                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 422
 423     def _select_api_hostname(self, req_api_hostname, default_client=None):
 424         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 425                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 426
 427     def _extract_api_key(self, ytcfg=None, default_client='web'):
 428         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 429
 430     def _extract_context(self, ytcfg=None, default_client='web'):
 431         context = get_first(
 432             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 433         # Enforce language and tz for extraction
 434         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 435         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 436         return context
 437
 438     _SAPISID = None
 439
 440     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 441         time_now = round(time.time())
 442         if self._SAPISID is None:
 443             yt_cookies = self._get_cookies('https://www.youtube.com')
 444             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 445             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 446             sapisid_cookie = dict_get(
 447                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 448             if sapisid_cookie and sapisid_cookie.value:
 449                 self._SAPISID = sapisid_cookie.value
 450                 self.write_debug('Extracted SAPISID cookie')
 451                 # SAPISID cookie is required if not already present
 452                 if not yt_cookies.get('SAPISID'):
 453                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 454                     self._set_cookie(
 455                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 456             else:
 457                 self._SAPISID = False
 458         if not self._SAPISID:
 459             return None
 460         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 461         sapisidhash = hashlib.sha1(
 462             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 463         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 464
 465     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 466                   note='Downloading API JSON', errnote='Unable to download API page',
 467                   context=None, api_key=None, api_hostname=None, default_client='web'):
 468
 469         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 470         data.update(query)
 471         real_headers = self.generate_api_headers(default_client=default_client)
 472         real_headers.update({'content-type': 'application/json'})
 473         if headers:
 474             real_headers.update(headers)
 475         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 476                    or api_key or self._extract_api_key(default_client=default_client))
 477         return self._download_json(
 478             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 479             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 480             data=json.dumps(data).encode('utf8'), headers=real_headers,
 481             query={'key': api_key, 'prettyPrint': 'false'})
 482
 483     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 484         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 485
 486     @staticmethod
 487     def _extract_session_index(*data):
 488         """
 489         Index of current account in account list.
 490         See: https://github.com/yt-dlp/yt-dlp/pull/519
 491         """
 492         for ytcfg in data:
 493             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 494             if session_index is not None:
 495                 return session_index
 496
 497     # Deprecated?
 498     def _extract_identity_token(self, ytcfg=None, webpage=None):
 499         if ytcfg:
 500             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 501             if token:
 502                 return token
 503         if webpage:
 504             return self._search_regex(
 505                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 506                 'identity token', default=None, fatal=False)
 507
 508     @staticmethod
 509     def _extract_account_syncid(*args):
 510         """
 511         Extract syncId required to download private playlists of secondary channels
 512         @params response and/or ytcfg
 513         """
 514         for data in args:
 515             # ytcfg includes channel_syncid if on secondary channel
 516             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 517             if delegated_sid:
 518                 return delegated_sid
 519             sync_ids = (try_get(
 520                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 521                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 522             if len(sync_ids) >= 2 and sync_ids[1]:
 523                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 524                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 525                 return sync_ids[0]
 526
 527     @staticmethod
 528     def _extract_visitor_data(*args):
 529         """
 530         Extracts visitorData from an API response or ytcfg
 531         Appears to be used to track session state
 532         """
 533         return get_first(
 534             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 535             expected_type=str)
 536
 537     @functools.cached_property
 538     def is_authenticated(self):
 539         return bool(self._generate_sapisidhash_header())
 540
 541     def extract_ytcfg(self, video_id, webpage):
 542         if not webpage:
 543             return {}
 544         return self._parse_json(
 545             self._search_regex(
 546                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 547                 default='{}'), video_id, fatal=False) or {}
 548
 549     def generate_api_headers(
 550             self, *, ytcfg=None, account_syncid=None, session_index=None,
 551             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 552
 553         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 554         headers = {
 555             'X-YouTube-Client-Name': compat_str(
 556                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 557             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 558             'Origin': origin,
 559             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 560             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 561             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 562         }
 563         if session_index is None:
 564             session_index = self._extract_session_index(ytcfg)
 565         if account_syncid or session_index is not None:
 566             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 567
 568         auth = self._generate_sapisidhash_header(origin)
 569         if auth is not None:
 570             headers['Authorization'] = auth
 571             headers['X-Origin'] = origin
 572         return {h: v for h, v in headers.items() if v is not None}
 573
 574     def _download_ytcfg(self, client, video_id):
 575         url = {
 576             'web': 'https://www.youtube.com',
 577             'web_music': 'https://music.youtube.com',
 578             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 579         }.get(client)
 580         if not url:
 581             return {}
 582         webpage = self._download_webpage(
 583             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 584         return self.extract_ytcfg(video_id, webpage) or {}
 585
 586     @staticmethod
 587     def _build_api_continuation_query(continuation, ctp=None):
 588         query = {
 589             'continuation': continuation
 590         }
 591         # TODO: Inconsistency with clickTrackingParams.
 592         # Currently we have a fixed ctp contained within context (from ytcfg)
 593         # and a ctp in root query for continuation.
 594         if ctp:
 595             query['clickTracking'] = {'clickTrackingParams': ctp}
 596         return query
 597
 598     @classmethod
 599     def _extract_next_continuation_data(cls, renderer):
 600         next_continuation = try_get(
 601             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 602                        lambda x: x['continuation']['reloadContinuationData']), dict)
 603         if not next_continuation:
 604             return
 605         continuation = next_continuation.get('continuation')
 606         if not continuation:
 607             return
 608         ctp = next_continuation.get('clickTrackingParams')
 609         return cls._build_api_continuation_query(continuation, ctp)
 610
 611     @classmethod
 612     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 613         if isinstance(continuation_ep, dict):
 614             continuation = try_get(
 615                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 616             if not continuation:
 617                 return
 618             ctp = continuation_ep.get('clickTrackingParams')
 619             return cls._build_api_continuation_query(continuation, ctp)
 620
 621     @classmethod
 622     def _extract_continuation(cls, renderer):
 623         next_continuation = cls._extract_next_continuation_data(renderer)
 624         if next_continuation:
 625             return next_continuation
 626
 627         contents = []
 628         for key in ('contents', 'items'):
 629             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 630
 631         for content in contents:
 632             if not isinstance(content, dict):
 633                 continue
 634             continuation_ep = try_get(
 635                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 636                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 637                 dict)
 638             continuation = cls._extract_continuation_ep_data(continuation_ep)
 639             if continuation:
 640                 return continuation
 641
 642     @classmethod
 643     def _extract_alerts(cls, data):
 644         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 645             if not isinstance(alert_dict, dict):
 646                 continue
 647             for alert in alert_dict.values():
 648                 alert_type = alert.get('type')
 649                 if not alert_type:
 650                     continue
 651                 message = cls._get_text(alert, 'text')
 652                 if message:
 653                     yield alert_type, message
 654
 655     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 656         errors = []
 657         warnings = []
 658         for alert_type, alert_message in alerts:
 659             if alert_type.lower() == 'error' and fatal:
 660                 errors.append([alert_type, alert_message])
 661             else:
 662                 warnings.append([alert_type, alert_message])
 663
 664         for alert_type, alert_message in (warnings + errors[:-1]):
 665             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 666         if errors:
 667             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 668
 669     def _extract_and_report_alerts(self, data, *args, **kwargs):
 670         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 671
 672     def _extract_badges(self, renderer: dict):
 673         badges = set()
 674         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 675             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 676             if label:
 677                 badges.add(label.lower())
 678         return badges
 679
 680     @staticmethod
 681     def _get_text(data, *path_list, max_runs=None):
 682         for path in path_list or [None]:
 683             if path is None:
 684                 obj = [data]
 685             else:
 686                 obj = traverse_obj(data, path, default=[])
 687                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 688                     obj = [obj]
 689             for item in obj:
 690                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 691                 if text:
 692                     return text
 693                 runs = try_get(item, lambda x: x['runs'], list) or []
 694                 if not runs and isinstance(item, list):
 695                     runs = item
 696
 697                 runs = runs[:min(len(runs), max_runs or len(runs))]
 698                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 699                 if text:
 700                     return text
 701
 702     def _get_count(self, data, *path_list):
 703         count_text = self._get_text(data, *path_list) or ''
 704         count = parse_count(count_text)
 705         if count is None:
 706             count = str_to_int(
 707                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 708         return count
 709
 710     @staticmethod
 711     def _extract_thumbnails(data, *path_list):
 712         """
 713         Extract thumbnails from thumbnails dict
 714         @param path_list: path list to level that contains 'thumbnails' key
 715         """
 716         thumbnails = []
 717         for path in path_list or [()]:
 718             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 719                 thumbnail_url = url_or_none(thumbnail.get('url'))
 720                 if not thumbnail_url:
 721                     continue
 722                 # Sometimes youtube gives a wrong thumbnail URL. See:
 723                 # https://github.com/yt-dlp/yt-dlp/issues/233
 724                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 725                 if 'maxresdefault' in thumbnail_url:
 726                     thumbnail_url = thumbnail_url.split('?')[0]
 727                 thumbnails.append({
 728                     'url': thumbnail_url,
 729                     'height': int_or_none(thumbnail.get('height')),
 730                     'width': int_or_none(thumbnail.get('width')),
 731                 })
 732         return thumbnails
 733
 734     @staticmethod
 735     def extract_relative_time(relative_time_text):
 736         """
 737         Extracts a relative time from string and converts to dt object
 738         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 739         """
 740         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 741         if mobj:
 742             start = mobj.group('start')
 743             if start:
 744                 return datetime_from_str(start)
 745             try:
 746                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 747             except ValueError:
 748                 return None
 749
 750     def _extract_time_text(self, renderer, *path_list):
 751         """@returns (timestamp, time_text)"""
 752         text = self._get_text(renderer, *path_list) or ''
 753         dt = self.extract_relative_time(text)
 754         timestamp = None
 755         if isinstance(dt, datetime.datetime):
 756             timestamp = calendar.timegm(dt.timetuple())
 757
 758         if timestamp is None:
 759             timestamp = (
 760                 unified_timestamp(text) or unified_timestamp(
 761                     self._search_regex(
 762                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 763                         text.lower(), 'time text', default=None)))
 764
 765         if text and timestamp is None:
 766             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 767         return timestamp, text
 768
 769     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 770                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 771                           default_client='web'):
 772         response = None
 773         last_error = None
 774         count = -1
 775         retries = self.get_param('extractor_retries', 3)
 776         if check_get_keys is None:
 777             check_get_keys = []
 778         while count < retries:
 779             count += 1
 780             if last_error:
 781                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 782             try:
 783                 response = self._call_api(
 784                     ep=ep, fatal=True, headers=headers,
 785                     video_id=item_id, query=query,
 786                     context=self._extract_context(ytcfg, default_client),
 787                     api_key=self._extract_api_key(ytcfg, default_client),
 788                     api_hostname=api_hostname, default_client=default_client,
 789                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 790             except ExtractorError as e:
 791                 if isinstance(e.cause, network_exceptions):
 792                     if isinstance(e.cause, compat_HTTPError):
 793                         first_bytes = e.cause.read(512)
 794                         if not is_html(first_bytes):
 795                             yt_error = try_get(
 796                                 self._parse_json(
 797                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 798                                 lambda x: x['error']['message'], compat_str)
 799                             if yt_error:
 800                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 801                     # Downloading page may result in intermittent 5xx HTTP error
 802                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 803                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 804                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 805                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 806                         last_error = error_to_compat_str(e.cause or e.msg)
 807                         if count < retries:
 808                             continue
 809                 if fatal:
 810                     raise
 811                 else:
 812                     self.report_warning(error_to_compat_str(e))
 813                     return
 814
 815             else:
 816                 try:
 817                     self._extract_and_report_alerts(response, only_once=True)
 818                 except ExtractorError as e:
 819                     # YouTube servers may return errors we want to retry on in a 200 OK response
 820                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 821                     if 'unknown error' in e.msg.lower():
 822                         last_error = e.msg
 823                         continue
 824                     if fatal:
 825                         raise
 826                     self.report_warning(error_to_compat_str(e))
 827                     return
 828                 if not check_get_keys or dict_get(response, check_get_keys):
 829                     break
 830                 # Youtube sometimes sends incomplete data
 831                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 832                 last_error = 'Incomplete data received'
 833                 if count >= retries:
 834                     if fatal:
 835                         raise ExtractorError(last_error)
 836                     else:
 837                         self.report_warning(last_error)
 838                         return
 839         return response
 840
 841     @staticmethod
 842     def is_music_url(url):
 843         return re.match(r'https?://music\.youtube\.com/', url) is not None
 844
 845     def _extract_video(self, renderer):
 846         video_id = renderer.get('videoId')
 847         title = self._get_text(renderer, 'title')
 848         description = self._get_text(renderer, 'descriptionSnippet')
 849         duration = parse_duration(self._get_text(
 850             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 851         if duration is None:
 852             duration = parse_duration(self._search_regex(
 853                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 854                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 855                 video_id, default=None, group='duration'))
 856
 857         view_count = self._get_count(renderer, 'viewCountText')
 858
 859         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 860         channel_id = traverse_obj(
 861             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 862             expected_type=str, get_all=False)
 863         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 864         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 865         overlay_style = traverse_obj(
 866             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 867             get_all=False, expected_type=str)
 868         badges = self._extract_badges(renderer)
 869         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 870         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 871             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 872             expected_type=str)) or ''
 873         url = f'https://www.youtube.com/watch?v={video_id}'
 874         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 875             url = f'https://www.youtube.com/shorts/{video_id}'
 876
 877         return {
 878             '_type': 'url',
 879             'ie_key': YoutubeIE.ie_key(),
 880             'id': video_id,
 881             'url': url,
 882             'title': title,
 883             'description': description,
 884             'duration': duration,
 885             'view_count': view_count,
 886             'uploader': uploader,
 887             'channel_id': channel_id,
 888             'thumbnails': thumbnails,
 889             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 890                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 891                             else None),
 892             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 893                             else 'was_live' if 'streamed' in time_text.lower()
 894                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 895                             else None),
 896             'release_timestamp': scheduled_timestamp,
 897             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 898         }
 899
 900
 901 class YoutubeIE(YoutubeBaseInfoExtractor):
 902     IE_DESC = 'YouTube'
 903     _VALID_URL = r"""(?x)^
 904                      (
 905                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 906                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 907                             (?:www\.)?deturl\.com/www\.youtube\.com|
 908                             (?:www\.)?pwnyoutube\.com|
 909                             (?:www\.)?hooktube\.com|
 910                             (?:www\.)?yourepeat\.com|
 911                             tube\.majestyc\.net|
 912                             %(invidious)s|
 913                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 914                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 915                          (?:                                                  # the various things that can precede the ID:
 916                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 917                              |(?:                                             # or the v= param in all its forms
 918                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 919                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 920                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 921                                  v=
 922                              )
 923                          ))
 924                          |(?:
 925                             youtu\.be|                                        # just youtu.be/xxxx
 926                             vid\.plus|                                        # or vid.plus/xxxx
 927                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 928                             %(invidious)s
 929                          )/
 930                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 931                          )
 932                      )?                                                       # all until now is optional -> you can pass the naked ID
 933                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 934                      (?(1).+)?                                                # if we found the ID, everything can follow
 935                      (?:\#|$)""" % {
 936         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 937     }
 938     _PLAYER_INFO_RE = (
 939         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 940         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 941         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 942     )
 943     _formats = {
 944         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 945         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 946         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 947         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 948         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 949         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 950         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 951         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 953         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 954         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 955         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 956         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 957         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 958         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 959         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 960         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 961         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 962
 963
 964         # 3D videos
 965         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 966         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 967         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 968         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 969         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 970         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 971         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 972
 973         # Apple HTTP Live Streaming
 974         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 975         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 976         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 977         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 978         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 979         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 980         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 981         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 982
 983         # DASH mp4 video
 984         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 985         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 986         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 988         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 989         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 990         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 991         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 992         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 993         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 994         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 995         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 996
 997         # Dash mp4 audio
 998         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 999         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1000         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1001         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1002         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1003         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1004         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1005
1006         # Dash webm
1007         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1008         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1009         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1010         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1011         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1012         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1013         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1014         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1015         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1016         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1017         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1018         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1020         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1021         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1022         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1023         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1024         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1025         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1026         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1027         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1028         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1029
1030         # Dash webm audio
1031         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1032         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1033
1034         # Dash webm audio with opus inside
1035         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1036         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1037         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1038
1039         # RTMP (unnamed)
1040         '_rtmp': {'protocol': 'rtmp'},
1041
1042         # av01 video only formats sometimes served with "unknown" codecs
1043         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1044         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1045         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1046         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1047         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1048         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1049         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1050         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1051     }
1052     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1053
1054     _GEO_BYPASS = False
1055
1056     IE_NAME = 'youtube'
1057     _TESTS = [
1058         {
1059             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1060             'info_dict': {
1061                 'id': 'BaW_jenozKc',
1062                 'ext': 'mp4',
1063                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1064                 'uploader': 'Philipp Hagemeister',
1065                 'uploader_id': 'phihag',
1066                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1067                 'channel': 'Philipp Hagemeister',
1068                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1069                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1070                 'upload_date': '20121002',
1071                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1072                 'categories': ['Science & Technology'],
1073                 'tags': ['youtube-dl'],
1074                 'duration': 10,
1075                 'view_count': int,
1076                 'like_count': int,
1077                 'availability': 'public',
1078                 'playable_in_embed': True,
1079                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1080                 'live_status': 'not_live',
1081                 'age_limit': 0,
1082                 'start_time': 1,
1083                 'end_time': 9,
1084                 'channel_follower_count': int
1085             }
1086         },
1087         {
1088             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1089             'note': 'Embed-only video (#1746)',
1090             'info_dict': {
1091                 'id': 'yZIXLfi8CZQ',
1092                 'ext': 'mp4',
1093                 'upload_date': '20120608',
1094                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1095                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1096                 'uploader': 'SET India',
1097                 'uploader_id': 'setindia',
1098                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1099                 'age_limit': 18,
1100             },
1101             'skip': 'Private video',
1102         },
1103         {
1104             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1105             'note': 'Use the first video ID in the URL',
1106             'info_dict': {
1107                 'id': 'BaW_jenozKc',
1108                 'ext': 'mp4',
1109                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1110                 'uploader': 'Philipp Hagemeister',
1111                 'uploader_id': 'phihag',
1112                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1113                 'channel': 'Philipp Hagemeister',
1114                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1115                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1116                 'upload_date': '20121002',
1117                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1118                 'categories': ['Science & Technology'],
1119                 'tags': ['youtube-dl'],
1120                 'duration': 10,
1121                 'view_count': int,
1122                 'like_count': int,
1123                 'availability': 'public',
1124                 'playable_in_embed': True,
1125                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1126                 'live_status': 'not_live',
1127                 'age_limit': 0,
1128                 'channel_follower_count': int
1129             },
1130             'params': {
1131                 'skip_download': True,
1132             },
1133         },
1134         {
1135             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1136             'note': '256k DASH audio (format 141) via DASH manifest',
1137             'info_dict': {
1138                 'id': 'a9LDPn-MO4I',
1139                 'ext': 'm4a',
1140                 'upload_date': '20121002',
1141                 'uploader_id': '8KVIDEO',
1142                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1143                 'description': '',
1144                 'uploader': '8KVIDEO',
1145                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1146             },
1147             'params': {
1148                 'youtube_include_dash_manifest': True,
1149                 'format': '141',
1150             },
1151             'skip': 'format 141 not served anymore',
1152         },
1153         # DASH manifest with encrypted signature
1154         {
1155             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1156             'info_dict': {
1157                 'id': 'IB3lcPjvWLA',
1158                 'ext': 'm4a',
1159                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1160                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1161                 'duration': 244,
1162                 'uploader': 'AfrojackVEVO',
1163                 'uploader_id': 'AfrojackVEVO',
1164                 'upload_date': '20131011',
1165                 'abr': 129.495,
1166                 'like_count': int,
1167                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1168                 'playable_in_embed': True,
1169                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1170                 'view_count': int,
1171                 'track': 'The Spark',
1172                 'live_status': 'not_live',
1173                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1174                 'channel': 'Afrojack',
1175                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1176                 'tags': 'count:19',
1177                 'availability': 'public',
1178                 'categories': ['Music'],
1179                 'age_limit': 0,
1180                 'alt_title': 'The Spark',
1181                 'channel_follower_count': int
1182             },
1183             'params': {
1184                 'youtube_include_dash_manifest': True,
1185                 'format': '141/bestaudio[ext=m4a]',
1186             },
1187         },
1188         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1189         {
1190             'note': 'Embed allowed age-gate video',
1191             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1192             'info_dict': {
1193                 'id': 'HtVdAasjOgU',
1194                 'ext': 'mp4',
1195                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1196                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1197                 'duration': 142,
1198                 'uploader': 'The Witcher',
1199                 'uploader_id': 'WitcherGame',
1200                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1201                 'upload_date': '20140605',
1202                 'age_limit': 18,
1203                 'categories': ['Gaming'],
1204                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1205                 'availability': 'needs_auth',
1206                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1207                 'like_count': int,
1208                 'channel': 'The Witcher',
1209                 'live_status': 'not_live',
1210                 'tags': 'count:17',
1211                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1212                 'playable_in_embed': True,
1213                 'view_count': int,
1214                 'channel_follower_count': int
1215             },
1216         },
1217         {
1218             'note': 'Age-gate video with embed allowed in public site',
1219             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1220             'info_dict': {
1221                 'id': 'HsUATh_Nc2U',
1222                 'ext': 'mp4',
1223                 'title': 'Godzilla 2 (Official Video)',
1224                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1225                 'upload_date': '20200408',
1226                 'uploader_id': 'FlyingKitty900',
1227                 'uploader': 'FlyingKitty',
1228                 'age_limit': 18,
1229                 'availability': 'needs_auth',
1230                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1231                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1232                 'channel': 'FlyingKitty',
1233                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1234                 'view_count': int,
1235                 'categories': ['Entertainment'],
1236                 'live_status': 'not_live',
1237                 'tags': ['Flyingkitty', 'godzilla 2'],
1238                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1239                 'like_count': int,
1240                 'duration': 177,
1241                 'playable_in_embed': True,
1242                 'channel_follower_count': int
1243             },
1244         },
1245         {
1246             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1247             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1248             'info_dict': {
1249                 'id': 'Tq92D6wQ1mg',
1250                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1251                 'ext': 'mp4',
1252                 'upload_date': '20191228',
1253                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1254                 'uploader': 'Projekt Melody',
1255                 'description': 'md5:17eccca93a786d51bc67646756894066',
1256                 'age_limit': 18,
1257                 'like_count': int,
1258                 'availability': 'needs_auth',
1259                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1260                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1261                 'view_count': int,
1262                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1263                 'channel': 'Projekt Melody',
1264                 'live_status': 'not_live',
1265                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1266                 'playable_in_embed': True,
1267                 'categories': ['Entertainment'],
1268                 'duration': 106,
1269                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1270                 'channel_follower_count': int
1271             },
1272         },
1273         {
1274             'note': 'Non-Agegated non-embeddable video',
1275             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1276             'info_dict': {
1277                 'id': 'MeJVWBSsPAY',
1278                 'ext': 'mp4',
1279                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1280                 'uploader': 'Herr Lurik',
1281                 'uploader_id': 'st3in234',
1282                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1283                 'upload_date': '20130730',
1284                 'track': 'Such mich find mich',
1285                 'age_limit': 0,
1286                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1287                 'like_count': int,
1288                 'playable_in_embed': False,
1289                 'creator': 'OOMPH!',
1290                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1291                 'view_count': int,
1292                 'alt_title': 'Such mich find mich',
1293                 'duration': 210,
1294                 'channel': 'Herr Lurik',
1295                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1296                 'categories': ['Music'],
1297                 'availability': 'public',
1298                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1299                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1300                 'live_status': 'not_live',
1301                 'artist': 'OOMPH!',
1302                 'channel_follower_count': int
1303             },
1304         },
1305         {
1306             'note': 'Non-bypassable age-gated video',
1307             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1308             'only_matching': True,
1309         },
1310         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1311         # YouTube Red ad is not captured for creator
1312         {
1313             'url': '__2ABJjxzNo',
1314             'info_dict': {
1315                 'id': '__2ABJjxzNo',
1316                 'ext': 'mp4',
1317                 'duration': 266,
1318                 'upload_date': '20100430',
1319                 'uploader_id': 'deadmau5',
1320                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1321                 'creator': 'deadmau5',
1322                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1323                 'uploader': 'deadmau5',
1324                 'title': 'Deadmau5 - Some Chords (HD)',
1325                 'alt_title': 'Some Chords',
1326                 'availability': 'public',
1327                 'tags': 'count:14',
1328                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1329                 'view_count': int,
1330                 'live_status': 'not_live',
1331                 'channel': 'deadmau5',
1332                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1333                 'like_count': int,
1334                 'track': 'Some Chords',
1335                 'artist': 'deadmau5',
1336                 'playable_in_embed': True,
1337                 'age_limit': 0,
1338                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1339                 'categories': ['Music'],
1340                 'album': 'Some Chords',
1341                 'channel_follower_count': int
1342             },
1343             'expected_warnings': [
1344                 'DASH manifest missing',
1345             ]
1346         },
1347         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1348         {
1349             'url': 'lqQg6PlCWgI',
1350             'info_dict': {
1351                 'id': 'lqQg6PlCWgI',
1352                 'ext': 'mp4',
1353                 'duration': 6085,
1354                 'upload_date': '20150827',
1355                 'uploader_id': 'olympic',
1356                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1357                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1358                 'uploader': 'Olympics',
1359                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1360                 'like_count': int,
1361                 'release_timestamp': 1343767800,
1362                 'playable_in_embed': True,
1363                 'categories': ['Sports'],
1364                 'release_date': '20120731',
1365                 'channel': 'Olympics',
1366                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1367                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1368                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1369                 'age_limit': 0,
1370                 'availability': 'public',
1371                 'live_status': 'was_live',
1372                 'view_count': int,
1373                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1374                 'channel_follower_count': int
1375             },
1376             'params': {
1377                 'skip_download': 'requires avconv',
1378             }
1379         },
1380         # Non-square pixels
1381         {
1382             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1383             'info_dict': {
1384                 'id': '_b-2C3KPAM0',
1385                 'ext': 'mp4',
1386                 'stretched_ratio': 16 / 9.,
1387                 'duration': 85,
1388                 'upload_date': '20110310',
1389                 'uploader_id': 'AllenMeow',
1390                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1391                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1392                 'uploader': '孫ᄋᄅ',
1393                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1394                 'playable_in_embed': True,
1395                 'channel': '孫ᄋᄅ',
1396                 'age_limit': 0,
1397                 'tags': 'count:11',
1398                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1399                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1400                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1401                 'view_count': int,
1402                 'categories': ['People & Blogs'],
1403                 'like_count': int,
1404                 'live_status': 'not_live',
1405                 'availability': 'unlisted',
1406                 'channel_follower_count': int
1407             },
1408         },
1409         # url_encoded_fmt_stream_map is empty string
1410         {
1411             'url': 'qEJwOuvDf7I',
1412             'info_dict': {
1413                 'id': 'qEJwOuvDf7I',
1414                 'ext': 'webm',
1415                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1416                 'description': '',
1417                 'upload_date': '20150404',
1418                 'uploader_id': 'spbelect',
1419                 'uploader': 'Наблюдатели Петербурга',
1420             },
1421             'params': {
1422                 'skip_download': 'requires avconv',
1423             },
1424             'skip': 'This live event has ended.',
1425         },
1426         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1427         {
1428             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1429             'info_dict': {
1430                 'id': 'FIl7x6_3R5Y',
1431                 'ext': 'webm',
1432                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1433                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1434                 'duration': 220,
1435                 'upload_date': '20150625',
1436                 'uploader_id': 'dorappi2000',
1437                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1438                 'uploader': 'dorappi2000',
1439                 'formats': 'mincount:31',
1440             },
1441             'skip': 'not actual anymore',
1442         },
1443         # DASH manifest with segment_list
1444         {
1445             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1446             'md5': '8ce563a1d667b599d21064e982ab9e31',
1447             'info_dict': {
1448                 'id': 'CsmdDsKjzN8',
1449                 'ext': 'mp4',
1450                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1451                 'uploader': 'Airtek',
1452                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1453                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1454                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1455             },
1456             'params': {
1457                 'youtube_include_dash_manifest': True,
1458                 'format': '135',  # bestvideo
1459             },
1460             'skip': 'This live event has ended.',
1461         },
1462         {
1463             # Multifeed videos (multiple cameras), URL is for Main Camera
1464             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1465             'info_dict': {
1466                 'id': 'jvGDaLqkpTg',
1467                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1468                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1469             },
1470             'playlist': [{
1471                 'info_dict': {
1472                     'id': 'jvGDaLqkpTg',
1473                     'ext': 'mp4',
1474                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1475                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1476                     'duration': 10643,
1477                     'upload_date': '20161111',
1478                     'uploader': 'Team PGP',
1479                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1480                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1481                 },
1482             }, {
1483                 'info_dict': {
1484                     'id': '3AKt1R1aDnw',
1485                     'ext': 'mp4',
1486                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1487                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1488                     'duration': 10991,
1489                     'upload_date': '20161111',
1490                     'uploader': 'Team PGP',
1491                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1492                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1493                 },
1494             }, {
1495                 'info_dict': {
1496                     'id': 'RtAMM00gpVc',
1497                     'ext': 'mp4',
1498                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1499                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1500                     'duration': 10995,
1501                     'upload_date': '20161111',
1502                     'uploader': 'Team PGP',
1503                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1504                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1505                 },
1506             }, {
1507                 'info_dict': {
1508                     'id': '6N2fdlP3C5U',
1509                     'ext': 'mp4',
1510                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1511                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1512                     'duration': 10990,
1513                     'upload_date': '20161111',
1514                     'uploader': 'Team PGP',
1515                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1516                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1517                 },
1518             }],
1519             'params': {
1520                 'skip_download': True,
1521             },
1522             'skip': 'Not multifeed anymore',
1523         },
1524         {
1525             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1526             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1527             'info_dict': {
1528                 'id': 'gVfLd0zydlo',
1529                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1530             },
1531             'playlist_count': 2,
1532             'skip': 'Not multifeed anymore',
1533         },
1534         {
1535             'url': 'https://vid.plus/FlRa-iH7PGw',
1536             'only_matching': True,
1537         },
1538         {
1539             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1540             'only_matching': True,
1541         },
1542         {
1543             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1544             # Also tests cut-off URL expansion in video description (see
1545             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1546             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1547             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1548             'info_dict': {
1549                 'id': 'lsguqyKfVQg',
1550                 'ext': 'mp4',
1551                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1552                 'alt_title': 'Dark Walk',
1553                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1554                 'duration': 133,
1555                 'upload_date': '20151119',
1556                 'uploader_id': 'IronSoulElf',
1557                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1558                 'uploader': 'IronSoulElf',
1559                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1560                 'track': 'Dark Walk',
1561                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1562                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1563                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1564                 'categories': ['Film & Animation'],
1565                 'view_count': int,
1566                 'live_status': 'not_live',
1567                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1568                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1569                 'tags': 'count:13',
1570                 'availability': 'public',
1571                 'channel': 'IronSoulElf',
1572                 'playable_in_embed': True,
1573                 'like_count': int,
1574                 'age_limit': 0,
1575                 'channel_follower_count': int
1576             },
1577             'params': {
1578                 'skip_download': True,
1579             },
1580         },
1581         {
1582             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1583             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1584             'only_matching': True,
1585         },
1586         {
1587             # Video with yt:stretch=17:0
1588             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1589             'info_dict': {
1590                 'id': 'Q39EVAstoRM',
1591                 'ext': 'mp4',
1592                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1593                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1594                 'upload_date': '20151107',
1595                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1596                 'uploader': 'CH GAMER DROID',
1597             },
1598             'params': {
1599                 'skip_download': True,
1600             },
1601             'skip': 'This video does not exist.',
1602         },
1603         {
1604             # Video with incomplete 'yt:stretch=16:'
1605             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1606             'only_matching': True,
1607         },
1608         {
1609             # Video licensed under Creative Commons
1610             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1611             'info_dict': {
1612                 'id': 'M4gD1WSo5mA',
1613                 'ext': 'mp4',
1614                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1615                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1616                 'duration': 721,
1617                 'upload_date': '20150128',
1618                 'uploader_id': 'BerkmanCenter',
1619                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1620                 'uploader': 'The Berkman Klein Center for Internet & Society',
1621                 'license': 'Creative Commons Attribution license (reuse allowed)',
1622                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1623                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1624                 'like_count': int,
1625                 'age_limit': 0,
1626                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1627                 'channel': 'The Berkman Klein Center for Internet & Society',
1628                 'availability': 'public',
1629                 'view_count': int,
1630                 'categories': ['Education'],
1631                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1632                 'live_status': 'not_live',
1633                 'playable_in_embed': True,
1634                 'channel_follower_count': int
1635             },
1636             'params': {
1637                 'skip_download': True,
1638             },
1639         },
1640         {
1641             # Channel-like uploader_url
1642             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1643             'info_dict': {
1644                 'id': 'eQcmzGIKrzg',
1645                 'ext': 'mp4',
1646                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1647                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1648                 'duration': 4060,
1649                 'upload_date': '20151120',
1650                 'uploader': 'Bernie Sanders',
1651                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1652                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1653                 'license': 'Creative Commons Attribution license (reuse allowed)',
1654                 'playable_in_embed': True,
1655                 'tags': 'count:12',
1656                 'like_count': int,
1657                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1658                 'age_limit': 0,
1659                 'availability': 'public',
1660                 'categories': ['News & Politics'],
1661                 'channel': 'Bernie Sanders',
1662                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1663                 'view_count': int,
1664                 'live_status': 'not_live',
1665                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1666                 'channel_follower_count': int
1667             },
1668             'params': {
1669                 'skip_download': True,
1670             },
1671         },
1672         {
1673             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1674             'only_matching': True,
1675         },
1676         {
1677             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1678             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1679             'only_matching': True,
1680         },
1681         {
1682             # Rental video preview
1683             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1684             'info_dict': {
1685                 'id': 'uGpuVWrhIzE',
1686                 'ext': 'mp4',
1687                 'title': 'Piku - Trailer',
1688                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1689                 'upload_date': '20150811',
1690                 'uploader': 'FlixMatrix',
1691                 'uploader_id': 'FlixMatrixKaravan',
1692                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1693                 'license': 'Standard YouTube License',
1694             },
1695             'params': {
1696                 'skip_download': True,
1697             },
1698             'skip': 'This video is not available.',
1699         },
1700         {
1701             # YouTube Red video with episode data
1702             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1703             'info_dict': {
1704                 'id': 'iqKdEhx-dD4',
1705                 'ext': 'mp4',
1706                 'title': 'Isolation - Mind Field (Ep 1)',
1707                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1708                 'duration': 2085,
1709                 'upload_date': '20170118',
1710                 'uploader': 'Vsauce',
1711                 'uploader_id': 'Vsauce',
1712                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1713                 'series': 'Mind Field',
1714                 'season_number': 1,
1715                 'episode_number': 1,
1716                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1717                 'tags': 'count:12',
1718                 'view_count': int,
1719                 'availability': 'public',
1720                 'age_limit': 0,
1721                 'channel': 'Vsauce',
1722                 'episode': 'Episode 1',
1723                 'categories': ['Entertainment'],
1724                 'season': 'Season 1',
1725                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1726                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1727                 'like_count': int,
1728                 'playable_in_embed': True,
1729                 'live_status': 'not_live',
1730                 'channel_follower_count': int
1731             },
1732             'params': {
1733                 'skip_download': True,
1734             },
1735             'expected_warnings': [
1736                 'Skipping DASH manifest',
1737             ],
1738         },
1739         {
1740             # The following content has been identified by the YouTube community
1741             # as inappropriate or offensive to some audiences.
1742             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1743             'info_dict': {
1744                 'id': '6SJNVb0GnPI',
1745                 'ext': 'mp4',
1746                 'title': 'Race Differences in Intelligence',
1747                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1748                 'duration': 965,
1749                 'upload_date': '20140124',
1750                 'uploader': 'New Century Foundation',
1751                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1752                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1753             },
1754             'params': {
1755                 'skip_download': True,
1756             },
1757             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1758         },
1759         {
1760             # itag 212
1761             'url': '1t24XAntNCY',
1762             'only_matching': True,
1763         },
1764         {
1765             # geo restricted to JP
1766             'url': 'sJL6WA-aGkQ',
1767             'only_matching': True,
1768         },
1769         {
1770             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1771             'only_matching': True,
1772         },
1773         {
1774             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1775             'only_matching': True,
1776         },
1777         {
1778             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1779             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1780             'only_matching': True,
1781         },
1782         {
1783             # DRM protected
1784             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1785             'only_matching': True,
1786         },
1787         {
1788             # Video with unsupported adaptive stream type formats
1789             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1790             'info_dict': {
1791                 'id': 'Z4Vy8R84T1U',
1792                 'ext': 'mp4',
1793                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1794                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1795                 'duration': 433,
1796                 'upload_date': '20130923',
1797                 'uploader': 'Amelia Putri Harwita',
1798                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1799                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1800                 'formats': 'maxcount:10',
1801             },
1802             'params': {
1803                 'skip_download': True,
1804                 'youtube_include_dash_manifest': False,
1805             },
1806             'skip': 'not actual anymore',
1807         },
1808         {
1809             # Youtube Music Auto-generated description
1810             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1811             'info_dict': {
1812                 'id': 'MgNrAu2pzNs',
1813                 'ext': 'mp4',
1814                 'title': 'Voyeur Girl',
1815                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1816                 'upload_date': '20190312',
1817                 'uploader': 'Stephen - Topic',
1818                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1819                 'artist': 'Stephen',
1820                 'track': 'Voyeur Girl',
1821                 'album': 'it\'s too much love to know my dear',
1822                 'release_date': '20190313',
1823                 'release_year': 2019,
1824                 'alt_title': 'Voyeur Girl',
1825                 'view_count': int,
1826                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1827                 'playable_in_embed': True,
1828                 'like_count': int,
1829                 'categories': ['Music'],
1830                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1831                 'channel': 'Stephen',
1832                 'availability': 'public',
1833                 'creator': 'Stephen',
1834                 'duration': 169,
1835                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1836                 'age_limit': 0,
1837                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1838                 'tags': 'count:11',
1839                 'live_status': 'not_live',
1840                 'channel_follower_count': int
1841             },
1842             'params': {
1843                 'skip_download': True,
1844             },
1845         },
1846         {
1847             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1848             'only_matching': True,
1849         },
1850         {
1851             # invalid -> valid video id redirection
1852             'url': 'DJztXj2GPfl',
1853             'info_dict': {
1854                 'id': 'DJztXj2GPfk',
1855                 'ext': 'mp4',
1856                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1857                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1858                 'upload_date': '20090125',
1859                 'uploader': 'Prochorowka',
1860                 'uploader_id': 'Prochorowka',
1861                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1862                 'artist': 'Panjabi MC',
1863                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1864                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1865             },
1866             'params': {
1867                 'skip_download': True,
1868             },
1869             'skip': 'Video unavailable',
1870         },
1871         {
1872             # empty description results in an empty string
1873             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1874             'info_dict': {
1875                 'id': 'x41yOUIvK2k',
1876                 'ext': 'mp4',
1877                 'title': 'IMG 3456',
1878                 'description': '',
1879                 'upload_date': '20170613',
1880                 'uploader_id': 'ElevageOrVert',
1881                 'uploader': 'ElevageOrVert',
1882                 'view_count': int,
1883                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1884                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1885                 'like_count': int,
1886                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1887                 'tags': [],
1888                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1889                 'availability': 'public',
1890                 'age_limit': 0,
1891                 'categories': ['Pets & Animals'],
1892                 'duration': 7,
1893                 'playable_in_embed': True,
1894                 'live_status': 'not_live',
1895                 'channel': 'ElevageOrVert',
1896                 'channel_follower_count': int
1897             },
1898             'params': {
1899                 'skip_download': True,
1900             },
1901         },
1902         {
1903             # with '};' inside yt initial data (see [1])
1904             # see [2] for an example with '};' inside ytInitialPlayerResponse
1905             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1906             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1907             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1908             'info_dict': {
1909                 'id': 'CHqg6qOn4no',
1910                 'ext': 'mp4',
1911                 'title': 'Part 77   Sort a list of simple types in c#',
1912                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1913                 'upload_date': '20130831',
1914                 'uploader_id': 'kudvenkat',
1915                 'uploader': 'kudvenkat',
1916                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1917                 'like_count': int,
1918                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1919                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1920                 'live_status': 'not_live',
1921                 'categories': ['Education'],
1922                 'availability': 'public',
1923                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1924                 'tags': 'count:12',
1925                 'playable_in_embed': True,
1926                 'age_limit': 0,
1927                 'view_count': int,
1928                 'duration': 522,
1929                 'channel': 'kudvenkat',
1930                 'channel_follower_count': int
1931             },
1932             'params': {
1933                 'skip_download': True,
1934             },
1935         },
1936         {
1937             # another example of '};' in ytInitialData
1938             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1939             'only_matching': True,
1940         },
1941         {
1942             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1943             'only_matching': True,
1944         },
1945         {
1946             # https://github.com/ytdl-org/youtube-dl/pull/28094
1947             'url': 'OtqTfy26tG0',
1948             'info_dict': {
1949                 'id': 'OtqTfy26tG0',
1950                 'ext': 'mp4',
1951                 'title': 'Burn Out',
1952                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1953                 'upload_date': '20141120',
1954                 'uploader': 'The Cinematic Orchestra - Topic',
1955                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1956                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1957                 'artist': 'The Cinematic Orchestra',
1958                 'track': 'Burn Out',
1959                 'album': 'Every Day',
1960                 'like_count': int,
1961                 'live_status': 'not_live',
1962                 'alt_title': 'Burn Out',
1963                 'duration': 614,
1964                 'age_limit': 0,
1965                 'view_count': int,
1966                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1967                 'creator': 'The Cinematic Orchestra',
1968                 'channel': 'The Cinematic Orchestra',
1969                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1970                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1971                 'availability': 'public',
1972                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1973                 'categories': ['Music'],
1974                 'playable_in_embed': True,
1975                 'channel_follower_count': int
1976             },
1977             'params': {
1978                 'skip_download': True,
1979             },
1980         },
1981         {
1982             # controversial video, only works with bpctr when authenticated with cookies
1983             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1984             'only_matching': True,
1985         },
1986         {
1987             # controversial video, requires bpctr/contentCheckOk
1988             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1989             'info_dict': {
1990                 'id': 'SZJvDhaSDnc',
1991                 'ext': 'mp4',
1992                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1993                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1994                 'uploader': 'CBS Mornings',
1995                 'uploader_id': 'CBSThisMorning',
1996                 'upload_date': '20140716',
1997                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1998                 'duration': 170,
1999                 'categories': ['News & Politics'],
2000                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2001                 'view_count': int,
2002                 'channel': 'CBS Mornings',
2003                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2004                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2005                 'age_limit': 18,
2006                 'availability': 'needs_auth',
2007                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2008                 'like_count': int,
2009                 'live_status': 'not_live',
2010                 'playable_in_embed': True,
2011                 'channel_follower_count': int
2012             }
2013         },
2014         {
2015             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2016             'url': 'cBvYw8_A0vQ',
2017             'info_dict': {
2018                 'id': 'cBvYw8_A0vQ',
2019                 'ext': 'mp4',
2020                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2021                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2022                 'upload_date': '20201120',
2023                 'uploader': 'Walk around Japan',
2024                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2025                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2026                 'duration': 1456,
2027                 'categories': ['Travel & Events'],
2028                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2029                 'view_count': int,
2030                 'channel': 'Walk around Japan',
2031                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2032                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2033                 'age_limit': 0,
2034                 'availability': 'public',
2035                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2036                 'live_status': 'not_live',
2037                 'playable_in_embed': True,
2038                 'channel_follower_count': int
2039             },
2040             'params': {
2041                 'skip_download': True,
2042             },
2043         }, {
2044             # Has multiple audio streams
2045             'url': 'WaOKSUlf4TM',
2046             'only_matching': True
2047         }, {
2048             # Requires Premium: has format 141 when requested using YTM url
2049             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2050             'only_matching': True
2051         }, {
2052             # multiple subtitles with same lang_code
2053             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2054             'only_matching': True,
2055         }, {
2056             # Force use android client fallback
2057             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2058             'info_dict': {
2059                 'id': 'YOelRv7fMxY',
2060                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2061                 'ext': '3gp',
2062                 'upload_date': '20210624',
2063                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2064                 'uploader': 'colinfurze',
2065                 'uploader_id': 'colinfurze',
2066                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2067                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2068                 'duration': 596,
2069                 'categories': ['Entertainment'],
2070                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2071                 'view_count': int,
2072                 'channel': 'colinfurze',
2073                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2074                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2075                 'age_limit': 0,
2076                 'availability': 'public',
2077                 'like_count': int,
2078                 'live_status': 'not_live',
2079                 'playable_in_embed': True,
2080                 'channel_follower_count': int
2081             },
2082             'params': {
2083                 'format': '17',  # 3gp format available on android
2084                 'extractor_args': {'youtube': {'player_client': ['android']}},
2085             },
2086         },
2087         {
2088             # Skip download of additional client configs (remix client config in this case)
2089             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2090             'only_matching': True,
2091             'params': {
2092                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2093             },
2094         }, {
2095             # shorts
2096             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2097             'only_matching': True,
2098         }, {
2099             'note': 'Storyboards',
2100             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2101             'info_dict': {
2102                 'id': '5KLPxDtMqe8',
2103                 'ext': 'mhtml',
2104                 'format_id': 'sb0',
2105                 'title': 'Your Brain is Plastic',
2106                 'uploader_id': 'scishow',
2107                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2108                 'upload_date': '20140324',
2109                 'uploader': 'SciShow',
2110                 'like_count': int,
2111                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2112                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2113                 'view_count': int,
2114                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2115                 'playable_in_embed': True,
2116                 'tags': 'count:12',
2117                 'uploader_url': 'http://www.youtube.com/user/scishow',
2118                 'availability': 'public',
2119                 'channel': 'SciShow',
2120                 'live_status': 'not_live',
2121                 'duration': 248,
2122                 'categories': ['Education'],
2123                 'age_limit': 0,
2124                 'channel_follower_count': int
2125             }, 'params': {'format': 'mhtml', 'skip_download': True}
2126         }, {
2127             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2128             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2129             'info_dict': {
2130                 'id': '2NUZ8W2llS4',
2131                 'ext': 'mp4',
2132                 'title': 'The NP that test your phone performance 🙂',
2133                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2134                 'uploader': 'Leon Nguyen',
2135                 'uploader_id': 'VNSXIII',
2136                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2137                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2138                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2139                 'duration': 21,
2140                 'view_count': int,
2141                 'age_limit': 0,
2142                 'categories': ['Gaming'],
2143                 'tags': 'count:23',
2144                 'playable_in_embed': True,
2145                 'live_status': 'not_live',
2146                 'upload_date': '20220103',
2147                 'like_count': int,
2148                 'availability': 'public',
2149                 'channel': 'Leon Nguyen',
2150                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2151                 'channel_follower_count': int
2152             }
2153         }, {
2154             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2155             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2156             'info_dict': {
2157                 'id': 'mzZzzBU6lrM',
2158                 'ext': 'mp4',
2159                 'title': 'I Met GeorgeNotFound In Real Life...',
2160                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2161                 'uploader': 'Quackity',
2162                 'uploader_id': 'QuackityHQ',
2163                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2164                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2165                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2166                 'duration': 955,
2167                 'view_count': int,
2168                 'age_limit': 0,
2169                 'categories': ['Entertainment'],
2170                 'tags': 'count:26',
2171                 'playable_in_embed': True,
2172                 'live_status': 'not_live',
2173                 'release_timestamp': 1641172509,
2174                 'release_date': '20220103',
2175                 'upload_date': '20220103',
2176                 'like_count': int,
2177                 'availability': 'public',
2178                 'channel': 'Quackity',
2179                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2180                 'channel_follower_count': int
2181             }
2182         },
2183         {   # continuous livestream. Microformat upload date should be preferred.
2184             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2185             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2186             'info_dict': {
2187                 'id': 'kgx4WGK0oNU',
2188                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2189                 'ext': 'mp4',
2190                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2191                 'availability': 'public',
2192                 'age_limit': 0,
2193                 'release_timestamp': 1637975704,
2194                 'upload_date': '20210619',
2195                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2196                 'live_status': 'is_live',
2197                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2198                 'uploader': '阿鲍Abao',
2199                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2200                 'channel': 'Abao in Tokyo',
2201                 'channel_follower_count': int,
2202                 'release_date': '20211127',
2203                 'tags': 'count:39',
2204                 'categories': ['People & Blogs'],
2205                 'like_count': int,
2206                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2207                 'view_count': int,
2208                 'playable_in_embed': True,
2209                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2210             },
2211             'params': {'skip_download': True}
2212         }, {
2213             # Story. Requires specific player params to work.
2214             # Note: stories get removed after some period of time
2215             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2216             'info_dict': {
2217                 'id': 'vv8qTUWmulI',
2218                 'ext': 'mp4',
2219                 'availability': 'unlisted',
2220                 'view_count': int,
2221                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2222                 'upload_date': '20220526',
2223                 'categories': ['Education'],
2224                 'title': 'Story',
2225                 'channel': 'IT\'S HISTORY',
2226                 'description': '',
2227                 'uploader_id': 'BlastfromthePast',
2228                 'duration': 12,
2229                 'uploader': 'IT\'S HISTORY',
2230                 'playable_in_embed': True,
2231                 'age_limit': 0,
2232                 'live_status': 'not_live',
2233                 'tags': [],
2234                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2235                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2236                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2237             }
2238         }, {
2239             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2240             'info_dict': {
2241                 'id': 'tjjjtzRLHvA',
2242                 'ext': 'mp4',
2243                 'title': 'ハッシュタグ無し };if window.ytcsi',
2244                 'upload_date': '20220323',
2245                 'like_count': int,
2246                 'availability': 'unlisted',
2247                 'channel': 'nao20010128nao',
2248                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2249                 'age_limit': 0,
2250                 'uploader': 'nao20010128nao',
2251                 'uploader_id': 'nao20010128nao',
2252                 'categories': ['Music'],
2253                 'view_count': int,
2254                 'description': '',
2255                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2256                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2257                 'live_status': 'not_live',
2258                 'playable_in_embed': True,
2259                 'channel_follower_count': int,
2260                 'duration': 6,
2261                 'tags': [],
2262                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2263             }
2264         }
2265     ]
2266
2267     @classmethod
2268     def suitable(cls, url):
2269         from ..utils import parse_qs
2270
2271         qs = parse_qs(url)
2272         if qs.get('list', [None])[0]:
2273             return False
2274         return super().suitable(url)
2275
2276     def __init__(self, *args, **kwargs):
2277         super().__init__(*args, **kwargs)
2278         self._code_cache = {}
2279         self._player_cache = {}
2280
2281     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2282         lock = threading.Lock()
2283
2284         is_live = True
2285         start_time = time.time()
2286         formats = [f for f in formats if f.get('is_from_start')]
2287
2288         def refetch_manifest(format_id, delay):
2289             nonlocal formats, start_time, is_live
2290             if time.time() <= start_time + delay:
2291                 return
2292
2293             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2294             video_details = traverse_obj(
2295                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2296             microformats = traverse_obj(
2297                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2298                 expected_type=dict, default=[])
2299             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2300             start_time = time.time()
2301
2302         def mpd_feed(format_id, delay):
2303             """
2304             @returns (manifest_url, manifest_stream_number, is_live) or None
2305             """
2306             with lock:
2307                 refetch_manifest(format_id, delay)
2308
2309             f = next((f for f in formats if f['format_id'] == format_id), None)
2310             if not f:
2311                 if not is_live:
2312                     self.to_screen(f'{video_id}: Video is no longer live')
2313                 else:
2314                     self.report_warning(
2315                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2316                 return None
2317             return f['manifest_url'], f['manifest_stream_number'], is_live
2318
2319         for f in formats:
2320             f['is_live'] = True
2321             f['protocol'] = 'http_dash_segments_generator'
2322             f['fragments'] = functools.partial(
2323                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2324
2325     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2326         FETCH_SPAN, MAX_DURATION = 5, 432000
2327
2328         mpd_url, stream_number, is_live = None, None, True
2329
2330         begin_index = 0
2331         download_start_time = ctx.get('start') or time.time()
2332
2333         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2334         if lack_early_segments:
2335             self.report_warning(bug_reports_message(
2336                 'Starting download from the last 120 hours of the live stream since '
2337                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2338             lack_early_segments = True
2339
2340         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2341         fragments, fragment_base_url = None, None
2342
2343         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2344             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2345             # Obtain from MPD's maximum seq value
2346             old_mpd_url = mpd_url
2347             last_error = ctx.pop('last_error', None)
2348             expire_fast = immediate or last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2349             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2350                                                or (mpd_url, stream_number, False))
2351             if not refresh_sequence:
2352                 if expire_fast and not is_live:
2353                     return False, last_seq
2354                 elif old_mpd_url == mpd_url:
2355                     return True, last_seq
2356             try:
2357                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2358                     mpd_url, None, note=False, errnote=False, fatal=False)
2359             except ExtractorError:
2360                 fmts = None
2361             if not fmts:
2362                 no_fragment_score += 2
2363                 return False, last_seq
2364             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2365             fragments = fmt_info['fragments']
2366             fragment_base_url = fmt_info['fragment_base_url']
2367             assert fragment_base_url
2368
2369             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2370             return True, _last_seq
2371
2372         while is_live:
2373             fetch_time = time.time()
2374             if no_fragment_score > 30:
2375                 return
2376             if last_segment_url:
2377                 # Obtain from "X-Head-Seqnum" header value from each segment
2378                 try:
2379                     urlh = self._request_webpage(
2380                         last_segment_url, None, note=False, errnote=False, fatal=False)
2381                 except ExtractorError:
2382                     urlh = None
2383                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2384                 if last_seq is None:
2385                     no_fragment_score += 2
2386                     last_segment_url = None
2387                     continue
2388             else:
2389                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2390                 no_fragment_score += 2
2391                 if not should_continue:
2392                     continue
2393
2394             if known_idx > last_seq:
2395                 last_segment_url = None
2396                 continue
2397
2398             last_seq += 1
2399
2400             if begin_index < 0 and known_idx < 0:
2401                 # skip from the start when it's negative value
2402                 known_idx = last_seq + begin_index
2403             if lack_early_segments:
2404                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2405             try:
2406                 for idx in range(known_idx, last_seq):
2407                     # do not update sequence here or you'll get skipped some part of it
2408                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2409                     if not should_continue:
2410                         known_idx = idx - 1
2411                         raise ExtractorError('breaking out of outer loop')
2412                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2413                     yield {
2414                         'url': last_segment_url,
2415                         'fragment_count': last_seq,
2416                     }
2417                 if known_idx == last_seq:
2418                     no_fragment_score += 5
2419                 else:
2420                     no_fragment_score = 0
2421                 known_idx = last_seq
2422             except ExtractorError:
2423                 continue
2424
2425             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2426
2427     def _extract_player_url(self, *ytcfgs, webpage=None):
2428         player_url = traverse_obj(
2429             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2430             get_all=False, expected_type=compat_str)
2431         if not player_url:
2432             return
2433         return urljoin('https://www.youtube.com', player_url)
2434
2435     def _download_player_url(self, video_id, fatal=False):
2436         res = self._download_webpage(
2437             'https://www.youtube.com/iframe_api',
2438             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2439         if res:
2440             player_version = self._search_regex(
2441                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2442             if player_version:
2443                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2444
2445     def _signature_cache_id(self, example_sig):
2446         """ Return a string representation of a signature """
2447         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2448
2449     @classmethod
2450     def _extract_player_info(cls, player_url):
2451         for player_re in cls._PLAYER_INFO_RE:
2452             id_m = re.search(player_re, player_url)
2453             if id_m:
2454                 break
2455         else:
2456             raise ExtractorError('Cannot identify player %r' % player_url)
2457         return id_m.group('id')
2458
2459     def _load_player(self, video_id, player_url, fatal=True):
2460         player_id = self._extract_player_info(player_url)
2461         if player_id not in self._code_cache:
2462             code = self._download_webpage(
2463                 player_url, video_id, fatal=fatal,
2464                 note='Downloading player ' + player_id,
2465                 errnote='Download of %s failed' % player_url)
2466             if code:
2467                 self._code_cache[player_id] = code
2468         return self._code_cache.get(player_id)
2469
2470     def _extract_signature_function(self, video_id, player_url, example_sig):
2471         player_id = self._extract_player_info(player_url)
2472
2473         # Read from filesystem cache
2474         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2475         assert os.path.basename(func_id) == func_id
2476
2477         cache_spec = self.cache.load('youtube-sigfuncs', func_id)
2478         if cache_spec is not None:
2479             return lambda s: ''.join(s[i] for i in cache_spec)
2480
2481         code = self._load_player(video_id, player_url)
2482         if code:
2483             res = self._parse_sig_js(code)
2484
2485             test_string = ''.join(map(chr, range(len(example_sig))))
2486             cache_res = res(test_string)
2487             cache_spec = [ord(c) for c in cache_res]
2488
2489             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2490             return res
2491
2492     def _print_sig_code(self, func, example_sig):
2493         if not self.get_param('youtube_print_sig_code'):
2494             return
2495
2496         def gen_sig_code(idxs):
2497             def _genslice(start, end, step):
2498                 starts = '' if start == 0 else str(start)
2499                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2500                 steps = '' if step == 1 else (':%d' % step)
2501                 return f's[{starts}{ends}{steps}]'
2502
2503             step = None
2504             # Quelch pyflakes warnings - start will be set when step is set
2505             start = '(Never used)'
2506             for i, prev in zip(idxs[1:], idxs[:-1]):
2507                 if step is not None:
2508                     if i - prev == step:
2509                         continue
2510                     yield _genslice(start, prev, step)
2511                     step = None
2512                     continue
2513                 if i - prev in [-1, 1]:
2514                     step = i - prev
2515                     start = prev
2516                     continue
2517                 else:
2518                     yield 's[%d]' % prev
2519             if step is None:
2520                 yield 's[%d]' % i
2521             else:
2522                 yield _genslice(start, i, step)
2523
2524         test_string = ''.join(map(chr, range(len(example_sig))))
2525         cache_res = func(test_string)
2526         cache_spec = [ord(c) for c in cache_res]
2527         expr_code = ' + '.join(gen_sig_code(cache_spec))
2528         signature_id_tuple = '(%s)' % (
2529             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2530         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2531                 '    return %s\n') % (signature_id_tuple, expr_code)
2532         self.to_screen('Extracted signature function:\n' + code)
2533
2534     def _parse_sig_js(self, jscode):
2535         funcname = self._search_regex(
2536             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2537              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2538              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2539              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2540              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2541              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2542              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2543              # Obsolete patterns
2544              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2545              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2546              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2547              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2548              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2549              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2550              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2551              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2552             jscode, 'Initial JS player signature function name', group='sig')
2553
2554         jsi = JSInterpreter(jscode)
2555         initial_function = jsi.extract_function(funcname)
2556         return lambda s: initial_function([s])
2557
2558     def _decrypt_signature(self, s, video_id, player_url):
2559         """Turn the encrypted s field into a working signature"""
2560         try:
2561             player_id = (player_url, self._signature_cache_id(s))
2562             if player_id not in self._player_cache:
2563                 func = self._extract_signature_function(video_id, player_url, s)
2564                 self._player_cache[player_id] = func
2565             func = self._player_cache[player_id]
2566             self._print_sig_code(func, s)
2567             return func(s)
2568         except Exception as e:
2569             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2570
2571     def _decrypt_nsig(self, s, video_id, player_url):
2572         """Turn the encrypted n field into a working signature"""
2573         if player_url is None:
2574             raise ExtractorError('Cannot decrypt nsig without player_url')
2575         player_url = urljoin('https://www.youtube.com', player_url)
2576
2577         sig_id = ('nsig_value', s)
2578         if sig_id in self._player_cache:
2579             return self._player_cache[sig_id]
2580
2581         try:
2582             player_id = ('nsig', player_url)
2583             if player_id not in self._player_cache:
2584                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2585             func = self._player_cache[player_id]
2586             self._player_cache[sig_id] = func(s)
2587             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2588             return self._player_cache[sig_id]
2589         except Exception as e:
2590             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2591
2592     def _extract_n_function_name(self, jscode):
2593         nfunc, idx = self._search_regex(
2594             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2595             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2596         if not idx:
2597             return nfunc
2598         return json.loads(js_to_json(self._search_regex(
2599             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2600             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2601
2602     def _extract_n_function(self, video_id, player_url):
2603         player_id = self._extract_player_info(player_url)
2604         func_code = self.cache.load('youtube-nsig', player_id)
2605
2606         if func_code:
2607             jsi = JSInterpreter(func_code)
2608         else:
2609             jscode = self._load_player(video_id, player_url)
2610             funcname = self._extract_n_function_name(jscode)
2611             jsi = JSInterpreter(jscode)
2612             func_code = jsi.extract_function_code(funcname)
2613             self.cache.store('youtube-nsig', player_id, func_code)
2614
2615         if self.get_param('youtube_print_sig_code'):
2616             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2617
2618         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2619
2620     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2621         """
2622         Extract signatureTimestamp (sts)
2623         Required to tell API what sig/player version is in use.
2624         """
2625         sts = None
2626         if isinstance(ytcfg, dict):
2627             sts = int_or_none(ytcfg.get('STS'))
2628
2629         if not sts:
2630             # Attempt to extract from player
2631             if player_url is None:
2632                 error_msg = 'Cannot extract signature timestamp without player_url.'
2633                 if fatal:
2634                     raise ExtractorError(error_msg)
2635                 self.report_warning(error_msg)
2636                 return
2637             code = self._load_player(video_id, player_url, fatal=fatal)
2638             if code:
2639                 sts = int_or_none(self._search_regex(
2640                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2641                     'JS player signature timestamp', group='sts', fatal=fatal))
2642         return sts
2643
2644     def _mark_watched(self, video_id, player_responses):
2645         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2646             label = 'fully ' if is_full else ''
2647             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2648                             expected_type=url_or_none)
2649             if not url:
2650                 self.report_warning(f'Unable to mark {label}watched')
2651                 return
2652             parsed_url = compat_urlparse.urlparse(url)
2653             qs = compat_urlparse.parse_qs(parsed_url.query)
2654
2655             # cpn generation algorithm is reverse engineered from base.js.
2656             # In fact it works even with dummy cpn.
2657             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2658             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2659
2660             # # more consistent results setting it to right before the end
2661             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2662
2663             qs.update({
2664                 'ver': ['2'],
2665                 'cpn': [cpn],
2666                 'cmt': video_length,
2667                 'el': 'detailpage',  # otherwise defaults to "shorts"
2668             })
2669
2670             if is_full:
2671                 # these seem to mark watchtime "history" in the real world
2672                 # they're required, so send in a single value
2673                 qs.update({
2674                     'st': video_length,
2675                     'et': video_length,
2676                 })
2677
2678             url = compat_urlparse.urlunparse(
2679                 parsed_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2680
2681             self._download_webpage(
2682                 url, video_id, f'Marking {label}watched',
2683                 'Unable to mark watched', fatal=False)
2684
2685     @staticmethod
2686     def _extract_urls(webpage):
2687         # Embedded YouTube player
2688         entries = [
2689             unescapeHTML(mobj.group('url'))
2690             for mobj in re.finditer(r'''(?x)
2691             (?:
2692                 <iframe[^>]+?src=|
2693                 data-video-url=|
2694                 <embed[^>]+?src=|
2695                 embedSWF\(?:\s*|
2696                 <object[^>]+data=|
2697                 new\s+SWFObject\(
2698             )
2699             (["\'])
2700                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2701                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2702             \1''', webpage)]
2703
2704         # lazyYT YouTube embed
2705         entries.extend(list(map(
2706             unescapeHTML,
2707             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2708
2709         # Wordpress "YouTube Video Importer" plugin
2710         matches = re.findall(r'''(?x)<div[^>]+
2711             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2712             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2713         entries.extend(m[-1] for m in matches)
2714
2715         return entries
2716
2717     @staticmethod
2718     def _extract_url(webpage):
2719         urls = YoutubeIE._extract_urls(webpage)
2720         return urls[0] if urls else None
2721
2722     @classmethod
2723     def extract_id(cls, url):
2724         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2725         if mobj is None:
2726             raise ExtractorError('Invalid URL: %s' % url)
2727         return mobj.group('id')
2728
2729     def _extract_chapters_from_json(self, data, duration):
2730         chapter_list = traverse_obj(
2731             data, (
2732                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2733                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2734             ), expected_type=list)
2735
2736         return self._extract_chapters(
2737             chapter_list,
2738             chapter_time=lambda chapter: float_or_none(
2739                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2740             chapter_title=lambda chapter: traverse_obj(
2741                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2742             duration=duration)
2743
2744     def _extract_chapters_from_engagement_panel(self, data, duration):
2745         content_list = traverse_obj(
2746             data,
2747             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2748             expected_type=list, default=[])
2749         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2750         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2751
2752         return next(filter(None, (
2753             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2754                                    chapter_time, chapter_title, duration)
2755             for contents in content_list)), [])
2756
2757     def _extract_chapters_from_description(self, description, duration):
2758         return self._extract_chapters(
2759             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
2760             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
2761             duration=duration, strict=False)
2762
2763     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
2764         if not duration:
2765             return
2766         chapter_list = [{
2767             'start_time': chapter_time(chapter),
2768             'title': chapter_title(chapter),
2769         } for chapter in chapter_list or []]
2770         if not strict:
2771             chapter_list.sort(key=lambda c: c['start_time'] or 0)
2772
2773         chapters = [{'start_time': 0, 'title': '<Untitled>'}]
2774         for idx, chapter in enumerate(chapter_list):
2775             if chapter['start_time'] is None or not chapter['title']:
2776                 self.report_warning(f'Incomplete chapter {idx}')
2777             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
2778                 chapters[-1]['end_time'] = chapter['start_time']
2779                 chapters.append(chapter)
2780             else:
2781                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
2782         chapters[-1]['end_time'] = duration
2783         return chapters if len(chapters) > 1 and chapters[1]['start_time'] else chapters[1:]
2784
2785     def _extract_comment(self, comment_renderer, parent=None):
2786         comment_id = comment_renderer.get('commentId')
2787         if not comment_id:
2788             return
2789
2790         text = self._get_text(comment_renderer, 'contentText')
2791
2792         # note: timestamp is an estimate calculated from the current time and time_text
2793         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2794         author = self._get_text(comment_renderer, 'authorText')
2795         author_id = try_get(comment_renderer,
2796                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2797
2798         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2799                                                        lambda x: x['likeCount']), compat_str)) or 0
2800         author_thumbnail = try_get(comment_renderer,
2801                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2802
2803         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2804         is_favorited = 'creatorHeart' in (try_get(
2805             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2806         return {
2807             'id': comment_id,
2808             'text': text,
2809             'timestamp': timestamp,
2810             'time_text': time_text,
2811             'like_count': votes,
2812             'is_favorited': is_favorited,
2813             'author': author,
2814             'author_id': author_id,
2815             'author_thumbnail': author_thumbnail,
2816             'author_is_uploader': author_is_uploader,
2817             'parent': parent or 'root'
2818         }
2819
2820     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2821
2822         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2823
2824         def extract_header(contents):
2825             _continuation = None
2826             for content in contents:
2827                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2828                 expected_comment_count = self._get_count(
2829                     comments_header_renderer, 'countText', 'commentsCount')
2830
2831                 if expected_comment_count:
2832                     tracker['est_total'] = expected_comment_count
2833                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2834                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2835
2836                 sort_menu_item = try_get(
2837                     comments_header_renderer,
2838                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2839                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2840
2841                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2842                 if not _continuation:
2843                     continue
2844
2845                 sort_text = str_or_none(sort_menu_item.get('title'))
2846                 if not sort_text:
2847                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2848                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2849                 break
2850             return _continuation
2851
2852         def extract_thread(contents):
2853             if not parent:
2854                 tracker['current_page_thread'] = 0
2855             for content in contents:
2856                 if not parent and tracker['total_parent_comments'] >= max_parents:
2857                     yield
2858                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2859                 comment_renderer = get_first(
2860                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2861                     expected_type=dict, default={})
2862
2863                 comment = self._extract_comment(comment_renderer, parent)
2864                 if not comment:
2865                     continue
2866
2867                 tracker['running_total'] += 1
2868                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2869                 yield comment
2870
2871                 # Attempt to get the replies
2872                 comment_replies_renderer = try_get(
2873                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2874
2875                 if comment_replies_renderer:
2876                     tracker['current_page_thread'] += 1
2877                     comment_entries_iter = self._comment_entries(
2878                         comment_replies_renderer, ytcfg, video_id,
2879                         parent=comment.get('id'), tracker=tracker)
2880                     yield from itertools.islice(comment_entries_iter, min(
2881                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2882
2883         # Keeps track of counts across recursive calls
2884         if not tracker:
2885             tracker = dict(
2886                 running_total=0,
2887                 est_total=0,
2888                 current_page_thread=0,
2889                 total_parent_comments=0,
2890                 total_reply_comments=0)
2891
2892         # TODO: Deprecated
2893         # YouTube comments have a max depth of 2
2894         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2895         if max_depth:
2896             self._downloader.deprecation_warning(
2897                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2898         if max_depth == 1 and parent:
2899             return
2900
2901         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2902             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2903
2904         continuation = self._extract_continuation(root_continuation_data)
2905
2906         response = None
2907         is_forced_continuation = False
2908         is_first_continuation = parent is None
2909         if is_first_continuation and not continuation:
2910             # Sometimes you can get comments by generating the continuation yourself,
2911             # even if YouTube initially reports them being disabled - e.g. stories comments.
2912             # Note: if the comment section is actually disabled, YouTube may return a response with
2913             # required check_get_keys missing. So we will disable that check initially in this case.
2914             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2915             is_forced_continuation = True
2916
2917         for page_num in itertools.count(0):
2918             if not continuation:
2919                 break
2920             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2921             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2922             if page_num == 0:
2923                 if is_first_continuation:
2924                     note_prefix = 'Downloading comment section API JSON'
2925                 else:
2926                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2927                         tracker['current_page_thread'], comment_prog_str)
2928             else:
2929                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2930                     '       ' if parent else '', ' replies' if parent else '',
2931                     page_num, comment_prog_str)
2932
2933             response = self._extract_response(
2934                 item_id=None, query=continuation,
2935                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2936                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2937             is_forced_continuation = False
2938             continuation_contents = traverse_obj(
2939                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2940
2941             continuation = None
2942             for continuation_section in continuation_contents:
2943                 continuation_items = traverse_obj(
2944                     continuation_section,
2945                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2946                     get_all=False, expected_type=list) or []
2947                 if is_first_continuation:
2948                     continuation = extract_header(continuation_items)
2949                     is_first_continuation = False
2950                     if continuation:
2951                         break
2952                     continue
2953
2954                 for entry in extract_thread(continuation_items):
2955                     if not entry:
2956                         return
2957                     yield entry
2958                 continuation = self._extract_continuation({'contents': continuation_items})
2959                 if continuation:
2960                     break
2961
2962         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2963         if message and not parent and tracker['running_total'] == 0:
2964             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2965
2966     @staticmethod
2967     def _generate_comment_continuation(video_id):
2968         """
2969         Generates initial comment section continuation token from given video id
2970         """
2971         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2972         return base64.b64encode(token.encode()).decode()
2973
2974     def _get_comments(self, ytcfg, video_id, contents, webpage):
2975         """Entry for comment extraction"""
2976         def _real_comment_extract(contents):
2977             renderer = next((
2978                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2979                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2980             yield from self._comment_entries(renderer, ytcfg, video_id)
2981
2982         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2983         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2984
2985     @staticmethod
2986     def _get_checkok_params():
2987         return {'contentCheckOk': True, 'racyCheckOk': True}
2988
2989     @classmethod
2990     def _generate_player_context(cls, sts=None):
2991         context = {
2992             'html5Preference': 'HTML5_PREF_WANTS',
2993         }
2994         if sts is not None:
2995             context['signatureTimestamp'] = sts
2996         return {
2997             'playbackContext': {
2998                 'contentPlaybackContext': context
2999             },
3000             **cls._get_checkok_params()
3001         }
3002
3003     @staticmethod
3004     def _is_agegated(player_response):
3005         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3006             return True
3007
3008         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3009         AGE_GATE_REASONS = (
3010             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3011             'age_verification_required', 'age_check_required',  # status
3012         )
3013         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3014
3015     @staticmethod
3016     def _is_unplayable(player_response):
3017         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3018
3019     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
3020
3021         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3022         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3023         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3024         headers = self.generate_api_headers(
3025             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3026
3027         yt_query = {
3028             'videoId': video_id,
3029             'params': '8AEB'  # enable stories
3030         }
3031         yt_query.update(self._generate_player_context(sts))
3032         return self._extract_response(
3033             item_id=video_id, ep='player', query=yt_query,
3034             ytcfg=player_ytcfg, headers=headers, fatal=True,
3035             default_client=client,
3036             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3037         ) or None
3038
3039     def _get_requested_clients(self, url, smuggled_data):
3040         requested_clients = []
3041         default = ['android', 'web']
3042         allowed_clients = sorted(
3043             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3044             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3045         for client in self._configuration_arg('player_client'):
3046             if client in allowed_clients:
3047                 requested_clients.append(client)
3048             elif client == 'default':
3049                 requested_clients.extend(default)
3050             elif client == 'all':
3051                 requested_clients.extend(allowed_clients)
3052             else:
3053                 self.report_warning(f'Skipping unsupported client {client}')
3054         if not requested_clients:
3055             requested_clients = default
3056
3057         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3058             requested_clients.extend(
3059                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3060
3061         return orderedSet(requested_clients)
3062
3063     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3064         initial_pr = None
3065         if webpage:
3066             initial_pr = self._search_json(
3067                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3068
3069         all_clients = set(clients)
3070         clients = clients[::-1]
3071         prs = []
3072
3073         def append_client(*client_names):
3074             """ Append the first client name that exists but not already used """
3075             for client_name in client_names:
3076                 actual_client = _split_innertube_client(client_name)[0]
3077                 if actual_client in INNERTUBE_CLIENTS:
3078                     if actual_client not in all_clients:
3079                         clients.append(client_name)
3080                         all_clients.add(actual_client)
3081                         return
3082
3083         # Android player_response does not have microFormats which are needed for
3084         # extraction of some data. So we return the initial_pr with formats
3085         # stripped out even if not requested by the user
3086         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3087         if initial_pr:
3088             pr = dict(initial_pr)
3089             pr['streamingData'] = None
3090             prs.append(pr)
3091
3092         last_error = None
3093         tried_iframe_fallback = False
3094         player_url = None
3095         while clients:
3096             client, base_client, variant = _split_innertube_client(clients.pop())
3097             player_ytcfg = master_ytcfg if client == 'web' else {}
3098             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3099                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3100
3101             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3102             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3103             if 'js' in self._configuration_arg('player_skip'):
3104                 require_js_player = False
3105                 player_url = None
3106
3107             if not player_url and not tried_iframe_fallback and require_js_player:
3108                 player_url = self._download_player_url(video_id)
3109                 tried_iframe_fallback = True
3110
3111             try:
3112                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3113                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3114             except ExtractorError as e:
3115                 if last_error:
3116                     self.report_warning(last_error)
3117                 last_error = e
3118                 continue
3119
3120             if pr:
3121                 prs.append(pr)
3122
3123             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3124             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3125                 append_client(f'{base_client}_creator')
3126             elif self._is_agegated(pr):
3127                 if variant == 'tv_embedded':
3128                     append_client(f'{base_client}_embedded')
3129                 elif not variant:
3130                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3131
3132         if last_error:
3133             if not len(prs):
3134                 raise last_error
3135             self.report_warning(last_error)
3136         return prs, player_url
3137
3138     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3139         itags, stream_ids = {}, []
3140         itag_qualities, res_qualities = {}, {}
3141         q = qualities([
3142             # Normally tiny is the smallest video-only formats. But
3143             # audio-only formats with unknown quality may get tagged as tiny
3144             'tiny',
3145             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3146             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3147         ])
3148         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3149
3150         for fmt in streaming_formats:
3151             if fmt.get('targetDurationSec'):
3152                 continue
3153
3154             itag = str_or_none(fmt.get('itag'))
3155             audio_track = fmt.get('audioTrack') or {}
3156             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3157             if stream_id in stream_ids:
3158                 continue
3159
3160             quality = fmt.get('quality')
3161             height = int_or_none(fmt.get('height'))
3162             if quality == 'tiny' or not quality:
3163                 quality = fmt.get('audioQuality', '').lower() or quality
3164             # The 3gp format (17) in android client has a quality of "small",
3165             # but is actually worse than other formats
3166             if itag == '17':
3167                 quality = 'tiny'
3168             if quality:
3169                 if itag:
3170                     itag_qualities[itag] = quality
3171                 if height:
3172                     res_qualities[height] = quality
3173             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3174             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3175             # number of fragment that would subsequently requested with (`&sq=N`)
3176             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3177                 continue
3178
3179             fmt_url = fmt.get('url')
3180             if not fmt_url:
3181                 sc = compat_parse_qs(fmt.get('signatureCipher'))
3182                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3183                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3184                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3185                     continue
3186                 try:
3187                     fmt_url += '&%s=%s' % (
3188                         traverse_obj(sc, ('sp', -1)) or 'signature',
3189                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3190                     )
3191                 except ExtractorError as e:
3192                     self.report_warning('Signature extraction failed: Some formats may be missing', only_once=True)
3193                     self.write_debug(e, only_once=True)
3194                     continue
3195
3196             query = parse_qs(fmt_url)
3197             throttled = False
3198             if query.get('n'):
3199                 try:
3200                     fmt_url = update_url_query(fmt_url, {
3201                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3202                 except ExtractorError as e:
3203                     self.report_warning(
3204                         'nsig extraction failed: You may experience throttling for some formats\n'
3205                         f'n = {query["n"][0]} ; player = {player_url}', only_once=True)
3206                     self.write_debug(e, only_once=True)
3207                     throttled = True
3208
3209             if itag:
3210                 itags[itag] = 'https'
3211                 stream_ids.append(stream_id)
3212
3213             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3214             language_preference = (
3215                 10 if audio_track.get('audioIsDefault') and 10
3216                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3217                 else -1)
3218             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3219             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3220             # Make sure to avoid false positives with small duration differences.
3221             # Eg: __2ABJjxzNo, ySuUZEjARPY
3222             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3223             if is_damaged:
3224                 self.report_warning(
3225                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3226             dct = {
3227                 'asr': int_or_none(fmt.get('audioSampleRate')),
3228                 'filesize': int_or_none(fmt.get('contentLength')),
3229                 'format_id': itag,
3230                 'format_note': join_nonempty(
3231                     '%s%s' % (audio_track.get('displayName') or '',
3232                               ' (default)' if language_preference > 0 else ''),
3233                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3234                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3235                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3236                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3237                 'fps': int_or_none(fmt.get('fps')) or None,
3238                 'height': height,
3239                 'quality': q(quality),
3240                 'has_drm': bool(fmt.get('drmFamilies')),
3241                 'tbr': tbr,
3242                 'url': fmt_url,
3243                 'width': int_or_none(fmt.get('width')),
3244                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3245                                           'desc' if language_preference < -1 else ''),
3246                 'language_preference': language_preference,
3247                 # Strictly de-prioritize damaged and 3gp formats
3248                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3249             }
3250             mime_mobj = re.match(
3251                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3252             if mime_mobj:
3253                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3254                 dct.update(parse_codecs(mime_mobj.group(2)))
3255             no_audio = dct.get('acodec') == 'none'
3256             no_video = dct.get('vcodec') == 'none'
3257             if no_audio:
3258                 dct['vbr'] = tbr
3259             if no_video:
3260                 dct['abr'] = tbr
3261             if no_audio or no_video:
3262                 dct['downloader_options'] = {
3263                     # Youtube throttles chunks >~10M
3264                     'http_chunk_size': 10485760,
3265                 }
3266                 if dct.get('ext'):
3267                     dct['container'] = dct['ext'] + '_dash'
3268             yield dct
3269
3270         live_from_start = is_live and self.get_param('live_from_start')
3271         skip_manifests = self._configuration_arg('skip')
3272         if not self.get_param('youtube_include_hls_manifest', True):
3273             skip_manifests.append('hls')
3274         if not self.get_param('youtube_include_dash_manifest', True):
3275             skip_manifests.append('dash')
3276         get_dash = 'dash' not in skip_manifests and (
3277             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3278         get_hls = not live_from_start and 'hls' not in skip_manifests
3279
3280         def process_manifest_format(f, proto, itag):
3281             if itag in itags:
3282                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3283                     return False
3284                 itag = f'{itag}-{proto}'
3285             if itag:
3286                 f['format_id'] = itag
3287                 itags[itag] = proto
3288
3289             f['quality'] = next((
3290                 q(qdict[val])
3291                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3292                 if val in qdict), -1)
3293             return True
3294
3295         for sd in streaming_data:
3296             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3297             if hls_manifest_url:
3298                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3299                     if process_manifest_format(f, 'hls', self._search_regex(
3300                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3301                         yield f
3302
3303             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3304             if dash_manifest_url:
3305                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3306                     if process_manifest_format(f, 'dash', f['format_id']):
3307                         f['filesize'] = int_or_none(self._search_regex(
3308                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3309                         if live_from_start:
3310                             f['is_from_start'] = True
3311
3312                         yield f
3313
3314     def _extract_storyboard(self, player_responses, duration):
3315         spec = get_first(
3316             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3317         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3318         if not base_url:
3319             return
3320         L = len(spec) - 1
3321         for i, args in enumerate(spec):
3322             args = args.split('#')
3323             counts = list(map(int_or_none, args[:5]))
3324             if len(args) != 8 or not all(counts):
3325                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3326                 continue
3327             width, height, frame_count, cols, rows = counts
3328             N, sigh = args[6:]
3329
3330             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3331             fragment_count = frame_count / (cols * rows)
3332             fragment_duration = duration / fragment_count
3333             yield {
3334                 'format_id': f'sb{i}',
3335                 'format_note': 'storyboard',
3336                 'ext': 'mhtml',
3337                 'protocol': 'mhtml',
3338                 'acodec': 'none',
3339                 'vcodec': 'none',
3340                 'url': url,
3341                 'width': width,
3342                 'height': height,
3343                 'fragments': [{
3344                     'url': url.replace('$M', str(j)),
3345                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3346                 } for j in range(math.ceil(fragment_count))],
3347             }
3348
3349     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3350         webpage = None
3351         if 'webpage' not in self._configuration_arg('player_skip'):
3352             webpage = self._download_webpage(
3353                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3354
3355         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3356
3357         player_responses, player_url = self._extract_player_responses(
3358             self._get_requested_clients(url, smuggled_data),
3359             video_id, webpage, master_ytcfg)
3360
3361         return webpage, master_ytcfg, player_responses, player_url
3362
3363     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3364         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3365         is_live = get_first(video_details, 'isLive')
3366         if is_live is None:
3367             is_live = get_first(live_broadcast_details, 'isLiveNow')
3368
3369         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3370         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3371
3372         return live_broadcast_details, is_live, streaming_data, formats
3373
3374     def _real_extract(self, url):
3375         url, smuggled_data = unsmuggle_url(url, {})
3376         video_id = self._match_id(url)
3377
3378         base_url = self.http_scheme() + '//www.youtube.com/'
3379         webpage_url = base_url + 'watch?v=' + video_id
3380
3381         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3382
3383         playability_statuses = traverse_obj(
3384             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3385
3386         trailer_video_id = get_first(
3387             playability_statuses,
3388             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3389             expected_type=str)
3390         if trailer_video_id:
3391             return self.url_result(
3392                 trailer_video_id, self.ie_key(), trailer_video_id)
3393
3394         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3395                        if webpage else (lambda x: None))
3396
3397         video_details = traverse_obj(
3398             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3399         microformats = traverse_obj(
3400             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3401             expected_type=dict, default=[])
3402         video_title = (
3403             get_first(video_details, 'title')
3404             or self._get_text(microformats, (..., 'title'))
3405             or search_meta(['og:title', 'twitter:title', 'title']))
3406         video_description = get_first(video_details, 'shortDescription')
3407
3408         multifeed_metadata_list = get_first(
3409             player_responses,
3410             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3411             expected_type=str)
3412         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3413             if self.get_param('noplaylist'):
3414                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3415             else:
3416                 entries = []
3417                 feed_ids = []
3418                 for feed in multifeed_metadata_list.split(','):
3419                     # Unquote should take place before split on comma (,) since textual
3420                     # fields may contain comma as well (see
3421                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3422                     feed_data = compat_parse_qs(
3423                         urllib.parse.unquote_plus(feed))
3424
3425                     def feed_entry(name):
3426                         return try_get(
3427                             feed_data, lambda x: x[name][0], compat_str)
3428
3429                     feed_id = feed_entry('id')
3430                     if not feed_id:
3431                         continue
3432                     feed_title = feed_entry('title')
3433                     title = video_title
3434                     if feed_title:
3435                         title += ' (%s)' % feed_title
3436                     entries.append({
3437                         '_type': 'url_transparent',
3438                         'ie_key': 'Youtube',
3439                         'url': smuggle_url(
3440                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3441                             {'force_singlefeed': True}),
3442                         'title': title,
3443                     })
3444                     feed_ids.append(feed_id)
3445                 self.to_screen(
3446                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3447                     % (', '.join(feed_ids), video_id))
3448                 return self.playlist_result(
3449                     entries, video_id, video_title, video_description)
3450
3451         duration = int_or_none(
3452             get_first(video_details, 'lengthSeconds')
3453             or get_first(microformats, 'lengthSeconds')
3454             or parse_duration(search_meta('duration'))) or None
3455
3456         if get_first(video_details, 'isPostLiveDvr'):
3457             self.write_debug('Video is in Post-Live Manifestless mode')
3458             if duration or 0 > 4 * 3600:
3459                 self.report_warning(
3460                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3461                     'This is a known issue and patches are welcome')
3462
3463         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3464             video_id, microformats, video_details, player_responses, player_url, duration)
3465
3466         if not formats:
3467             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3468                 self.report_drm(video_id)
3469             pemr = get_first(
3470                 playability_statuses,
3471                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3472             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3473             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3474             if subreason:
3475                 if subreason == 'The uploader has not made this video available in your country.':
3476                     countries = get_first(microformats, 'availableCountries')
3477                     if not countries:
3478                         regions_allowed = search_meta('regionsAllowed')
3479                         countries = regions_allowed.split(',') if regions_allowed else None
3480                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3481                 reason += f'. {subreason}'
3482             if reason:
3483                 self.raise_no_formats(reason, expected=True)
3484
3485         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3486         if not keywords and webpage:
3487             keywords = [
3488                 unescapeHTML(m.group('content'))
3489                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3490         for keyword in keywords:
3491             if keyword.startswith('yt:stretch='):
3492                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3493                 if mobj:
3494                     # NB: float is intentional for forcing float division
3495                     w, h = (float(v) for v in mobj.groups())
3496                     if w > 0 and h > 0:
3497                         ratio = w / h
3498                         for f in formats:
3499                             if f.get('vcodec') != 'none':
3500                                 f['stretched_ratio'] = ratio
3501                         break
3502         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3503         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3504         if thumbnail_url:
3505             thumbnails.append({
3506                 'url': thumbnail_url,
3507             })
3508         original_thumbnails = thumbnails.copy()
3509
3510         # The best resolution thumbnails sometimes does not appear in the webpage
3511         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3512         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3513         thumbnail_names = [
3514             # While the *1,*2,*3 thumbnails are just below their correspnding "*default" variants
3515             # in resolution, these are not the custom thumbnail. So de-prioritize them
3516             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3517             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3518         ]
3519         n_thumbnail_names = len(thumbnail_names)
3520         thumbnails.extend({
3521             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3522                 video_id=video_id, name=name, ext=ext,
3523                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3524         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3525         for thumb in thumbnails:
3526             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3527             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3528         self._remove_duplicate_formats(thumbnails)
3529         self._downloader._sort_thumbnails(original_thumbnails)
3530
3531         category = get_first(microformats, 'category') or search_meta('genre')
3532         channel_id = str_or_none(
3533             get_first(video_details, 'channelId')
3534             or get_first(microformats, 'externalChannelId')
3535             or search_meta('channelId'))
3536         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3537
3538         live_content = get_first(video_details, 'isLiveContent')
3539         is_upcoming = get_first(video_details, 'isUpcoming')
3540         if is_live is None:
3541             if is_upcoming or live_content is False:
3542                 is_live = False
3543         if is_upcoming is None and (live_content or is_live):
3544             is_upcoming = False
3545         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3546         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3547         if not duration and live_end_time and live_start_time:
3548             duration = live_end_time - live_start_time
3549
3550         if is_live and self.get_param('live_from_start'):
3551             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3552
3553         formats.extend(self._extract_storyboard(player_responses, duration))
3554
3555         # Source is given priority since formats that throttle are given lower source_preference
3556         # When throttling issue is fully fixed, remove this
3557         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3558
3559         info = {
3560             'id': video_id,
3561             'title': video_title,
3562             'formats': formats,
3563             'thumbnails': thumbnails,
3564             # The best thumbnail that we are sure exists. Prevents unnecessary
3565             # URL checking if user don't care about getting the best possible thumbnail
3566             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3567             'description': video_description,
3568             'uploader': get_first(video_details, 'author'),
3569             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3570             'uploader_url': owner_profile_url,
3571             'channel_id': channel_id,
3572             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3573             'duration': duration,
3574             'view_count': int_or_none(
3575                 get_first((video_details, microformats), (..., 'viewCount'))
3576                 or search_meta('interactionCount')),
3577             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3578             'age_limit': 18 if (
3579                 get_first(microformats, 'isFamilySafe') is False
3580                 or search_meta('isFamilyFriendly') == 'false'
3581                 or search_meta('og:restrictions:age') == '18+') else 0,
3582             'webpage_url': webpage_url,
3583             'categories': [category] if category else None,
3584             'tags': keywords,
3585             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3586             'is_live': is_live,
3587             'was_live': (False if is_live or is_upcoming or live_content is False
3588                          else None if is_live is None or is_upcoming is None
3589                          else live_content),
3590             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3591             'release_timestamp': live_start_time,
3592         }
3593
3594         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3595         if pctr:
3596             def get_lang_code(track):
3597                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3598                         or track.get('languageCode'))
3599
3600             # Converted into dicts to remove duplicates
3601             captions = {
3602                 get_lang_code(sub): sub
3603                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3604             translation_languages = {
3605                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3606                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3607
3608             def process_language(container, base_url, lang_code, sub_name, query):
3609                 lang_subs = container.setdefault(lang_code, [])
3610                 for fmt in self._SUBTITLE_FORMATS:
3611                     query.update({
3612                         'fmt': fmt,
3613                     })
3614                     lang_subs.append({
3615                         'ext': fmt,
3616                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3617                         'name': sub_name,
3618                     })
3619
3620             subtitles, automatic_captions = {}, {}
3621             for lang_code, caption_track in captions.items():
3622                 base_url = caption_track.get('baseUrl')
3623                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3624                 if not base_url:
3625                     continue
3626                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3627                 if caption_track.get('kind') != 'asr':
3628                     if not lang_code:
3629                         continue
3630                     process_language(
3631                         subtitles, base_url, lang_code, lang_name, {})
3632                     if not caption_track.get('isTranslatable'):
3633                         continue
3634                 for trans_code, trans_name in translation_languages.items():
3635                     if not trans_code:
3636                         continue
3637                     orig_trans_code = trans_code
3638                     if caption_track.get('kind') != 'asr':
3639                         if 'translated_subs' in self._configuration_arg('skip'):
3640                             continue
3641                         trans_code += f'-{lang_code}'
3642                         trans_name += format_field(lang_name, None, ' from %s')
3643                     # Add an "-orig" label to the original language so that it can be distinguished.
3644                     # The subs are returned without "-orig" as well for compatibility
3645                     if lang_code == f'a-{orig_trans_code}':
3646                         process_language(
3647                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3648                     # Setting tlang=lang returns damaged subtitles.
3649                     process_language(automatic_captions, base_url, trans_code, trans_name,
3650                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3651             info['automatic_captions'] = automatic_captions
3652             info['subtitles'] = subtitles
3653
3654         parsed_url = compat_urllib_parse_urlparse(url)
3655         for component in [parsed_url.fragment, parsed_url.query]:
3656             query = compat_parse_qs(component)
3657             for k, v in query.items():
3658                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3659                     d_k += '_time'
3660                     if d_k not in info and k in s_ks:
3661                         info[d_k] = parse_duration(query[k][0])
3662
3663         # Youtube Music Auto-generated description
3664         if video_description:
3665             mobj = re.search(
3666                 r'''(?xs)
3667                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3668                     (?P<album>[^\n]+)
3669                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3670                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3671                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3672                     .+\nAuto-generated\ by\ YouTube\.\s*$
3673                 ''', video_description)
3674             if mobj:
3675                 release_year = mobj.group('release_year')
3676                 release_date = mobj.group('release_date')
3677                 if release_date:
3678                     release_date = release_date.replace('-', '')
3679                     if not release_year:
3680                         release_year = release_date[:4]
3681                 info.update({
3682                     'album': mobj.group('album'.strip()),
3683                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3684                     'track': mobj.group('track').strip(),
3685                     'release_date': release_date,
3686                     'release_year': int_or_none(release_year),
3687                 })
3688
3689         initial_data = None
3690         if webpage:
3691             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
3692         if not initial_data:
3693             query = {'videoId': video_id}
3694             query.update(self._get_checkok_params())
3695             initial_data = self._extract_response(
3696                 item_id=video_id, ep='next', fatal=False,
3697                 ytcfg=master_ytcfg, query=query,
3698                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3699                 note='Downloading initial data API JSON')
3700
3701         info['comment_count'] = traverse_obj(initial_data, (
3702             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
3703             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
3704         ), (
3705             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
3706             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
3707         ), expected_type=int_or_none, get_all=False)
3708
3709         try:  # This will error if there is no livechat
3710             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3711         except (KeyError, IndexError, TypeError):
3712             pass
3713         else:
3714             info.setdefault('subtitles', {})['live_chat'] = [{
3715                 # url is needed to set cookies
3716                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
3717                 'video_id': video_id,
3718                 'ext': 'json',
3719                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3720             }]
3721
3722         if initial_data:
3723             info['chapters'] = (
3724                 self._extract_chapters_from_json(initial_data, duration)
3725                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3726                 or self._extract_chapters_from_description(video_description, duration)
3727                 or None)
3728
3729         contents = traverse_obj(
3730             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3731             expected_type=list, default=[])
3732
3733         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3734         if vpir:
3735             stl = vpir.get('superTitleLink')
3736             if stl:
3737                 stl = self._get_text(stl)
3738                 if try_get(
3739                         vpir,
3740                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3741                     info['location'] = stl
3742                 else:
3743                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3744                     if mobj:
3745                         info.update({
3746                             'series': mobj.group(1),
3747                             'season_number': int(mobj.group(2)),
3748                             'episode_number': int(mobj.group(3)),
3749                         })
3750             for tlb in (try_get(
3751                     vpir,
3752                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3753                     list) or []):
3754                 tbr = tlb.get('toggleButtonRenderer') or {}
3755                 for getter, regex in [(
3756                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3757                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3758                             lambda x: x['accessibility'],
3759                             lambda x: x['accessibilityData']['accessibilityData'],
3760                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3761                     label = (try_get(tbr, getter, dict) or {}).get('label')
3762                     if label:
3763                         mobj = re.match(regex, label)
3764                         if mobj:
3765                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3766                             break
3767             sbr_tooltip = try_get(
3768                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3769             if sbr_tooltip:
3770                 like_count, dislike_count = sbr_tooltip.split(' / ')
3771                 info.update({
3772                     'like_count': str_to_int(like_count),
3773                     'dislike_count': str_to_int(dislike_count),
3774                 })
3775         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3776         if vsir:
3777             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3778             info.update({
3779                 'channel': self._get_text(vor, 'title'),
3780                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3781
3782             rows = try_get(
3783                 vsir,
3784                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3785                 list) or []
3786             multiple_songs = False
3787             for row in rows:
3788                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3789                     multiple_songs = True
3790                     break
3791             for row in rows:
3792                 mrr = row.get('metadataRowRenderer') or {}
3793                 mrr_title = mrr.get('title')
3794                 if not mrr_title:
3795                     continue
3796                 mrr_title = self._get_text(mrr, 'title')
3797                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3798                 if mrr_title == 'License':
3799                     info['license'] = mrr_contents_text
3800                 elif not multiple_songs:
3801                     if mrr_title == 'Album':
3802                         info['album'] = mrr_contents_text
3803                     elif mrr_title == 'Artist':
3804                         info['artist'] = mrr_contents_text
3805                     elif mrr_title == 'Song':
3806                         info['track'] = mrr_contents_text
3807
3808         fallbacks = {
3809             'channel': 'uploader',
3810             'channel_id': 'uploader_id',
3811             'channel_url': 'uploader_url',
3812         }
3813
3814         # The upload date for scheduled, live and past live streams / premieres in microformats
3815         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3816         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3817         upload_date = (
3818             unified_strdate(get_first(microformats, 'uploadDate'))
3819             or unified_strdate(search_meta('uploadDate')))
3820         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3821             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3822         info['upload_date'] = upload_date
3823
3824         for to, frm in fallbacks.items():
3825             if not info.get(to):
3826                 info[to] = info.get(frm)
3827
3828         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3829             v = info.get(s_k)
3830             if v:
3831                 info[d_k] = v
3832
3833         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3834         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3835         is_membersonly = None
3836         is_premium = None
3837         if initial_data and is_private is not None:
3838             is_membersonly = False
3839             is_premium = False
3840             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3841             badge_labels = set()
3842             for content in contents:
3843                 if not isinstance(content, dict):
3844                     continue
3845                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3846             for badge_label in badge_labels:
3847                 if badge_label.lower() == 'members only':
3848                     is_membersonly = True
3849                 elif badge_label.lower() == 'premium':
3850                     is_premium = True
3851                 elif badge_label.lower() == 'unlisted':
3852                     is_unlisted = True
3853
3854         info['availability'] = self._availability(
3855             is_private=is_private,
3856             needs_premium=is_premium,
3857             needs_subscription=is_membersonly,
3858             needs_auth=info['age_limit'] >= 18,
3859             is_unlisted=None if is_private is None else is_unlisted)
3860
3861         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3862
3863         self.mark_watched(video_id, player_responses)
3864
3865         return info
3866
3867
3868 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3869
3870     @staticmethod
3871     def passthrough_smuggled_data(func):
3872         def _smuggle(entries, smuggled_data):
3873             for entry in entries:
3874                 # TODO: Convert URL to music.youtube instead.
3875                 # Do we need to passthrough any other smuggled_data?
3876                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3877                 yield entry
3878
3879         @functools.wraps(func)
3880         def wrapper(self, url):
3881             url, smuggled_data = unsmuggle_url(url, {})
3882             if self.is_music_url(url):
3883                 smuggled_data['is_music_url'] = True
3884             info_dict = func(self, url, smuggled_data)
3885             if smuggled_data and info_dict.get('entries'):
3886                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3887             return info_dict
3888         return wrapper
3889
3890     def _extract_channel_id(self, webpage):
3891         channel_id = self._html_search_meta(
3892             'channelId', webpage, 'channel id', default=None)
3893         if channel_id:
3894             return channel_id
3895         channel_url = self._html_search_meta(
3896             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3897              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3898              'twitter:app:url:googleplay'), webpage, 'channel url')
3899         return self._search_regex(
3900             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3901             channel_url, 'channel id')
3902
3903     @staticmethod
3904     def _extract_basic_item_renderer(item):
3905         # Modified from _extract_grid_item_renderer
3906         known_basic_renderers = (
3907             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3908         )
3909         for key, renderer in item.items():
3910             if not isinstance(renderer, dict):
3911                 continue
3912             elif key in known_basic_renderers:
3913                 return renderer
3914             elif key.startswith('grid') and key.endswith('Renderer'):
3915                 return renderer
3916
3917     def _grid_entries(self, grid_renderer):
3918         for item in grid_renderer['items']:
3919             if not isinstance(item, dict):
3920                 continue
3921             renderer = self._extract_basic_item_renderer(item)
3922             if not isinstance(renderer, dict):
3923                 continue
3924             title = self._get_text(renderer, 'title')
3925
3926             # playlist
3927             playlist_id = renderer.get('playlistId')
3928             if playlist_id:
3929                 yield self.url_result(
3930                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3931                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3932                     video_title=title)
3933                 continue
3934             # video
3935             video_id = renderer.get('videoId')
3936             if video_id:
3937                 yield self._extract_video(renderer)
3938                 continue
3939             # channel
3940             channel_id = renderer.get('channelId')
3941             if channel_id:
3942                 yield self.url_result(
3943                     'https://www.youtube.com/channel/%s' % channel_id,
3944                     ie=YoutubeTabIE.ie_key(), video_title=title)
3945                 continue
3946             # generic endpoint URL support
3947             ep_url = urljoin('https://www.youtube.com/', try_get(
3948                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3949                 compat_str))
3950             if ep_url:
3951                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3952                     if ie.suitable(ep_url):
3953                         yield self.url_result(
3954                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3955                         break
3956
3957     def _music_reponsive_list_entry(self, renderer):
3958         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3959         if video_id:
3960             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3961                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3962         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3963         if playlist_id:
3964             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3965             if video_id:
3966                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3967                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3968             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3969                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3970         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3971         if browse_id:
3972             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3973                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3974
3975     def _shelf_entries_from_content(self, shelf_renderer):
3976         content = shelf_renderer.get('content')
3977         if not isinstance(content, dict):
3978             return
3979         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3980         if renderer:
3981             # TODO: add support for nested playlists so each shelf is processed
3982             # as separate playlist
3983             # TODO: this includes only first N items
3984             yield from self._grid_entries(renderer)
3985         renderer = content.get('horizontalListRenderer')
3986         if renderer:
3987             # TODO
3988             pass
3989
3990     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3991         ep = try_get(
3992             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3993             compat_str)
3994         shelf_url = urljoin('https://www.youtube.com', ep)
3995         if shelf_url:
3996             # Skipping links to another channels, note that checking for
3997             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3998             # will not work
3999             if skip_channels and '/channels?' in shelf_url:
4000                 return
4001             title = self._get_text(shelf_renderer, 'title')
4002             yield self.url_result(shelf_url, video_title=title)
4003         # Shelf may not contain shelf URL, fallback to extraction from content
4004         yield from self._shelf_entries_from_content(shelf_renderer)
4005
4006     def _playlist_entries(self, video_list_renderer):
4007         for content in video_list_renderer['contents']:
4008             if not isinstance(content, dict):
4009                 continue
4010             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4011             if not isinstance(renderer, dict):
4012                 continue
4013             video_id = renderer.get('videoId')
4014             if not video_id:
4015                 continue
4016             yield self._extract_video(renderer)
4017
4018     def _rich_entries(self, rich_grid_renderer):
4019         renderer = try_get(
4020             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
4021         video_id = renderer.get('videoId')
4022         if not video_id:
4023             return
4024         yield self._extract_video(renderer)
4025
4026     def _video_entry(self, video_renderer):
4027         video_id = video_renderer.get('videoId')
4028         if video_id:
4029             return self._extract_video(video_renderer)
4030
4031     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4032         url = urljoin('https://youtube.com', traverse_obj(
4033             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4034         if url:
4035             return self.url_result(
4036                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4037
4038     def _post_thread_entries(self, post_thread_renderer):
4039         post_renderer = try_get(
4040             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4041         if not post_renderer:
4042             return
4043         # video attachment
4044         video_renderer = try_get(
4045             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4046         video_id = video_renderer.get('videoId')
4047         if video_id:
4048             entry = self._extract_video(video_renderer)
4049             if entry:
4050                 yield entry
4051         # playlist attachment
4052         playlist_id = try_get(
4053             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
4054         if playlist_id:
4055             yield self.url_result(
4056                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4057                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4058         # inline video links
4059         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4060         for run in runs:
4061             if not isinstance(run, dict):
4062                 continue
4063             ep_url = try_get(
4064                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
4065             if not ep_url:
4066                 continue
4067             if not YoutubeIE.suitable(ep_url):
4068                 continue
4069             ep_video_id = YoutubeIE._match_id(ep_url)
4070             if video_id == ep_video_id:
4071                 continue
4072             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4073
4074     def _post_thread_continuation_entries(self, post_thread_continuation):
4075         contents = post_thread_continuation.get('contents')
4076         if not isinstance(contents, list):
4077             return
4078         for content in contents:
4079             renderer = content.get('backstagePostThreadRenderer')
4080             if isinstance(renderer, dict):
4081                 yield from self._post_thread_entries(renderer)
4082                 continue
4083             renderer = content.get('videoRenderer')
4084             if isinstance(renderer, dict):
4085                 yield self._video_entry(renderer)
4086
4087     r''' # unused
4088     def _rich_grid_entries(self, contents):
4089         for content in contents:
4090             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4091             if video_renderer:
4092                 entry = self._video_entry(video_renderer)
4093                 if entry:
4094                     yield entry
4095     '''
4096
4097     def _extract_entries(self, parent_renderer, continuation_list):
4098         # continuation_list is modified in-place with continuation_list = [continuation_token]
4099         continuation_list[:] = [None]
4100         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4101         for content in contents:
4102             if not isinstance(content, dict):
4103                 continue
4104             is_renderer = traverse_obj(
4105                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4106                 expected_type=dict)
4107             if not is_renderer:
4108                 renderer = content.get('richItemRenderer')
4109                 if renderer:
4110                     for entry in self._rich_entries(renderer):
4111                         yield entry
4112                     continuation_list[0] = self._extract_continuation(parent_renderer)
4113                 continue
4114             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4115             for isr_content in isr_contents:
4116                 if not isinstance(isr_content, dict):
4117                     continue
4118
4119                 known_renderers = {
4120                     'playlistVideoListRenderer': self._playlist_entries,
4121                     'gridRenderer': self._grid_entries,
4122                     'reelShelfRenderer': self._grid_entries,
4123                     'shelfRenderer': self._shelf_entries,
4124                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4125                     'backstagePostThreadRenderer': self._post_thread_entries,
4126                     'videoRenderer': lambda x: [self._video_entry(x)],
4127                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4128                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4129                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4130                 }
4131                 for key, renderer in isr_content.items():
4132                     if key not in known_renderers:
4133                         continue
4134                     for entry in known_renderers[key](renderer):
4135                         if entry:
4136                             yield entry
4137                     continuation_list[0] = self._extract_continuation(renderer)
4138                     break
4139
4140             if not continuation_list[0]:
4141                 continuation_list[0] = self._extract_continuation(is_renderer)
4142
4143         if not continuation_list[0]:
4144             continuation_list[0] = self._extract_continuation(parent_renderer)
4145
4146     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4147         continuation_list = [None]
4148         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4149         tab_content = try_get(tab, lambda x: x['content'], dict)
4150         if not tab_content:
4151             return
4152         parent_renderer = (
4153             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4154             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4155         yield from extract_entries(parent_renderer)
4156         continuation = continuation_list[0]
4157
4158         for page_num in itertools.count(1):
4159             if not continuation:
4160                 break
4161             headers = self.generate_api_headers(
4162                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4163             response = self._extract_response(
4164                 item_id=f'{item_id} page {page_num}',
4165                 query=continuation, headers=headers, ytcfg=ytcfg,
4166                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4167
4168             if not response:
4169                 break
4170             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4171             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4172             visitor_data = self._extract_visitor_data(response) or visitor_data
4173
4174             known_continuation_renderers = {
4175                 'playlistVideoListContinuation': self._playlist_entries,
4176                 'gridContinuation': self._grid_entries,
4177                 'itemSectionContinuation': self._post_thread_continuation_entries,
4178                 'sectionListContinuation': extract_entries,  # for feeds
4179             }
4180             continuation_contents = try_get(
4181                 response, lambda x: x['continuationContents'], dict) or {}
4182             continuation_renderer = None
4183             for key, value in continuation_contents.items():
4184                 if key not in known_continuation_renderers:
4185                     continue
4186                 continuation_renderer = value
4187                 continuation_list = [None]
4188                 yield from known_continuation_renderers[key](continuation_renderer)
4189                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4190                 break
4191             if continuation_renderer:
4192                 continue
4193
4194             known_renderers = {
4195                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4196                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4197                 'gridVideoRenderer': (self._grid_entries, 'items'),
4198                 'gridChannelRenderer': (self._grid_entries, 'items'),
4199                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4200                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4201                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4202                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4203             }
4204             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4205             continuation_items = try_get(
4206                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4207             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4208             video_items_renderer = None
4209             for key, value in continuation_item.items():
4210                 if key not in known_renderers:
4211                     continue
4212                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4213                 continuation_list = [None]
4214                 yield from known_renderers[key][0](video_items_renderer)
4215                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4216                 break
4217             if video_items_renderer:
4218                 continue
4219             break
4220
4221     @staticmethod
4222     def _extract_selected_tab(tabs, fatal=True):
4223         for tab in tabs:
4224             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4225             if renderer.get('selected') is True:
4226                 return renderer
4227         else:
4228             if fatal:
4229                 raise ExtractorError('Unable to find selected tab')
4230
4231     def _extract_uploader(self, data):
4232         uploader = {}
4233         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4234         owner = try_get(
4235             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4236         if owner:
4237             owner_text = owner.get('text')
4238             uploader['uploader'] = self._search_regex(
4239                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4240             uploader['uploader_id'] = try_get(
4241                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
4242             uploader['uploader_url'] = urljoin(
4243                 'https://www.youtube.com/',
4244                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
4245         return {k: v for k, v in uploader.items() if v is not None}
4246
4247     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4248         playlist_id = title = description = channel_url = channel_name = channel_id = None
4249         tags = []
4250
4251         selected_tab = self._extract_selected_tab(tabs)
4252         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4253         renderer = try_get(
4254             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4255         if renderer:
4256             channel_name = renderer.get('title')
4257             channel_url = renderer.get('channelUrl')
4258             channel_id = renderer.get('externalId')
4259         else:
4260             renderer = try_get(
4261                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4262
4263         if renderer:
4264             title = renderer.get('title')
4265             description = renderer.get('description', '')
4266             playlist_id = channel_id
4267             tags = renderer.get('keywords', '').split()
4268
4269         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4270         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4271         def _get_uncropped(url):
4272             return url_or_none((url or '').split('=')[0] + '=s0')
4273
4274         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4275         if avatar_thumbnails:
4276             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4277             if uncropped_avatar:
4278                 avatar_thumbnails.append({
4279                     'url': uncropped_avatar,
4280                     'id': 'avatar_uncropped',
4281                     'preference': 1
4282                 })
4283
4284         channel_banners = self._extract_thumbnails(
4285             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4286         for banner in channel_banners:
4287             banner['preference'] = -10
4288
4289         if channel_banners:
4290             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4291             if uncropped_banner:
4292                 channel_banners.append({
4293                     'url': uncropped_banner,
4294                     'id': 'banner_uncropped',
4295                     'preference': -5
4296                 })
4297
4298         primary_thumbnails = self._extract_thumbnails(
4299             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4300
4301         if playlist_id is None:
4302             playlist_id = item_id
4303
4304         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4305         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4306         if title is None:
4307             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4308         title += format_field(selected_tab, 'title', ' - %s')
4309         title += format_field(selected_tab, 'expandedText', ' - %s')
4310
4311         metadata = {
4312             'playlist_id': playlist_id,
4313             'playlist_title': title,
4314             'playlist_description': description,
4315             'uploader': channel_name,
4316             'uploader_id': channel_id,
4317             'uploader_url': channel_url,
4318             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4319             'tags': tags,
4320             'view_count': self._get_count(playlist_stats, 1),
4321             'availability': self._extract_availability(data),
4322             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4323             'playlist_count': self._get_count(playlist_stats, 0),
4324             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4325         }
4326         if not channel_id:
4327             metadata.update(self._extract_uploader(data))
4328         metadata.update({
4329             'channel': metadata['uploader'],
4330             'channel_id': metadata['uploader_id'],
4331             'channel_url': metadata['uploader_url']})
4332         return self.playlist_result(
4333             self._entries(
4334                 selected_tab, playlist_id, ytcfg,
4335                 self._extract_account_syncid(ytcfg, data),
4336                 self._extract_visitor_data(data, ytcfg)),
4337             **metadata)
4338
4339     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4340         first_id = last_id = response = None
4341         for page_num in itertools.count(1):
4342             videos = list(self._playlist_entries(playlist))
4343             if not videos:
4344                 return
4345             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4346             if start >= len(videos):
4347                 return
4348             yield from videos[start:]
4349             first_id = first_id or videos[0]['id']
4350             last_id = videos[-1]['id']
4351             watch_endpoint = try_get(
4352                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4353             headers = self.generate_api_headers(
4354                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4355                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4356             query = {
4357                 'playlistId': playlist_id,
4358                 'videoId': watch_endpoint.get('videoId') or last_id,
4359                 'index': watch_endpoint.get('index') or len(videos),
4360                 'params': watch_endpoint.get('params') or 'OAE%3D'
4361             }
4362             response = self._extract_response(
4363                 item_id='%s page %d' % (playlist_id, page_num),
4364                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4365                 check_get_keys='contents'
4366             )
4367             playlist = try_get(
4368                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4369
4370     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4371         title = playlist.get('title') or try_get(
4372             data, lambda x: x['titleText']['simpleText'], compat_str)
4373         playlist_id = playlist.get('playlistId') or item_id
4374
4375         # Delegating everything except mix playlists to regular tab-based playlist URL
4376         playlist_url = urljoin(url, try_get(
4377             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4378             compat_str))
4379
4380         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4381         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4382         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4383
4384         if playlist_url and playlist_url != url and not is_known_unviewable:
4385             return self.url_result(
4386                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4387                 video_title=title)
4388
4389         return self.playlist_result(
4390             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4391             playlist_id=playlist_id, playlist_title=title)
4392
4393     def _extract_availability(self, data):
4394         """
4395         Gets the availability of a given playlist/tab.
4396         Note: Unless YouTube tells us explicitly, we do not assume it is public
4397         @param data: response
4398         """
4399         is_private = is_unlisted = None
4400         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4401         badge_labels = self._extract_badges(renderer)
4402
4403         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4404         privacy_dropdown_entries = try_get(
4405             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4406         for renderer_dict in privacy_dropdown_entries:
4407             is_selected = try_get(
4408                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4409             if not is_selected:
4410                 continue
4411             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4412             if label:
4413                 badge_labels.add(label.lower())
4414                 break
4415
4416         for badge_label in badge_labels:
4417             if badge_label == 'unlisted':
4418                 is_unlisted = True
4419             elif badge_label == 'private':
4420                 is_private = True
4421             elif badge_label == 'public':
4422                 is_unlisted = is_private = False
4423         return self._availability(is_private, False, False, False, is_unlisted)
4424
4425     @staticmethod
4426     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4427         sidebar_renderer = try_get(
4428             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4429         for item in sidebar_renderer:
4430             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4431             if renderer:
4432                 return renderer
4433
4434     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4435         """
4436         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4437         """
4438         browse_id = params = None
4439         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4440         if not renderer:
4441             return
4442         menu_renderer = try_get(
4443             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4444         for menu_item in menu_renderer:
4445             if not isinstance(menu_item, dict):
4446                 continue
4447             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4448             text = try_get(
4449                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4450             if not text or text.lower() != 'show unavailable videos':
4451                 continue
4452             browse_endpoint = try_get(
4453                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4454             browse_id = browse_endpoint.get('browseId')
4455             params = browse_endpoint.get('params')
4456             break
4457
4458         headers = self.generate_api_headers(
4459             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4460             visitor_data=self._extract_visitor_data(data, ytcfg))
4461         query = {
4462             'params': params or 'wgYCCAA=',
4463             'browseId': browse_id or 'VL%s' % item_id
4464         }
4465         return self._extract_response(
4466             item_id=item_id, headers=headers, query=query,
4467             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4468             note='Downloading API JSON with unavailable videos')
4469
4470     @functools.cached_property
4471     def skip_webpage(self):
4472         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4473
4474     def _extract_webpage(self, url, item_id, fatal=True):
4475         retries = self.get_param('extractor_retries', 3)
4476         count = -1
4477         webpage = data = last_error = None
4478         while count < retries:
4479             count += 1
4480             # Sometimes youtube returns a webpage with incomplete ytInitialData
4481             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4482             if last_error:
4483                 self.report_warning('%s. Retrying ...' % last_error)
4484             try:
4485                 webpage = self._download_webpage(
4486                     url, item_id,
4487                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4488                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4489             except ExtractorError as e:
4490                 if isinstance(e.cause, network_exceptions):
4491                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4492                         last_error = error_to_compat_str(e.cause or e.msg)
4493                         if count < retries:
4494                             continue
4495                 if fatal:
4496                     raise
4497                 self.report_warning(error_to_compat_str(e))
4498                 break
4499             else:
4500                 try:
4501                     self._extract_and_report_alerts(data)
4502                 except ExtractorError as e:
4503                     if fatal:
4504                         raise
4505                     self.report_warning(error_to_compat_str(e))
4506                     break
4507
4508                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4509                     break
4510
4511                 last_error = 'Incomplete yt initial data received'
4512                 if count >= retries:
4513                     if fatal:
4514                         raise ExtractorError(last_error)
4515                     self.report_warning(last_error)
4516                     break
4517
4518         return webpage, data
4519
4520     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4521         """Use if failed to extract ytcfg (and data) from initial webpage"""
4522         if not ytcfg and self.is_authenticated:
4523             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4524             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4525                 raise ExtractorError(
4526                     f'{msg}. If you are not downloading private content, or '
4527                     'your cookies are only for the first account and channel,'
4528                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4529                     expected=True)
4530             self.report_warning(msg, only_once=True)
4531
4532     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4533         data = None
4534         if not self.skip_webpage:
4535             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4536             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4537             # Reject webpage data if redirected to home page without explicitly requesting
4538             selected_tab = self._extract_selected_tab(traverse_obj(
4539                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4540             if (url != 'https://www.youtube.com/feed/recommended'
4541                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4542                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4543                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4544                 if fatal:
4545                     raise ExtractorError(msg, expected=True)
4546                 self.report_warning(msg, only_once=True)
4547         if not data:
4548             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4549             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4550         return data, ytcfg
4551
4552     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4553         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4554         resolve_response = self._extract_response(
4555             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4556             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4557         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4558         for ep_key, ep in endpoints.items():
4559             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4560             if params:
4561                 return self._extract_response(
4562                     item_id=item_id, query=params, ep=ep, headers=headers,
4563                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4564                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4565         err_note = 'Failed to resolve url (does the playlist exist?)'
4566         if fatal:
4567             raise ExtractorError(err_note, expected=True)
4568         self.report_warning(err_note, item_id)
4569
4570     _SEARCH_PARAMS = None
4571
4572     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4573         data = {'query': query}
4574         if params is NO_DEFAULT:
4575             params = self._SEARCH_PARAMS
4576         if params:
4577             data['params'] = params
4578
4579         content_keys = (
4580             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4581             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4582             # ytmusic search
4583             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4584             ('continuationContents', ),
4585         )
4586         display_id = f'query "{query}"'
4587         check_get_keys = tuple({keys[0] for keys in content_keys})
4588         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4589         self._report_playlist_authcheck(ytcfg, fatal=False)
4590
4591         continuation_list = [None]
4592         search = None
4593         for page_num in itertools.count(1):
4594             data.update(continuation_list[0] or {})
4595             headers = self.generate_api_headers(
4596                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4597             search = self._extract_response(
4598                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4599                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4600             slr_contents = traverse_obj(search, *content_keys)
4601             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4602             if not continuation_list[0]:
4603                 break
4604
4605
4606 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4607     IE_DESC = 'YouTube Tabs'
4608     _VALID_URL = r'''(?x:
4609         https?://
4610             (?:\w+\.)?
4611             (?:
4612                 youtube(?:kids)?\.com|
4613                 %(invidious)s
4614             )/
4615             (?:
4616                 (?P<channel_type>channel|c|user|browse)/|
4617                 (?P<not_channel>
4618                     feed/|hashtag/|
4619                     (?:playlist|watch)\?.*?\blist=
4620                 )|
4621                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4622             )
4623             (?P<id>[^/?\#&]+)
4624     )''' % {
4625         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4626         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4627     }
4628     IE_NAME = 'youtube:tab'
4629
4630     _TESTS = [{
4631         'note': 'playlists, multipage',
4632         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4633         'playlist_mincount': 94,
4634         'info_dict': {
4635             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4636             'title': 'Igor Kleiner - Playlists',
4637             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4638             'uploader': 'Igor Kleiner',
4639             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4640             'channel': 'Igor Kleiner',
4641             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4642             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4643             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4644             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4645             'channel_follower_count': int
4646         },
4647     }, {
4648         'note': 'playlists, multipage, different order',
4649         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4650         'playlist_mincount': 94,
4651         'info_dict': {
4652             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4653             'title': 'Igor Kleiner - Playlists',
4654             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4655             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4656             'uploader': 'Igor Kleiner',
4657             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4658             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4659             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4660             'channel': 'Igor Kleiner',
4661             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4662             'channel_follower_count': int
4663         },
4664     }, {
4665         'note': 'playlists, series',
4666         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4667         'playlist_mincount': 5,
4668         'info_dict': {
4669             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4670             'title': '3Blue1Brown - Playlists',
4671             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4672             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4673             'uploader': '3Blue1Brown',
4674             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4675             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4676             'channel': '3Blue1Brown',
4677             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4678             'tags': ['Mathematics'],
4679             'channel_follower_count': int
4680         },
4681     }, {
4682         'note': 'playlists, singlepage',
4683         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4684         'playlist_mincount': 4,
4685         'info_dict': {
4686             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4687             'title': 'ThirstForScience - Playlists',
4688             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4689             'uploader': 'ThirstForScience',
4690             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4691             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4692             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4693             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4694             'tags': 'count:13',
4695             'channel': 'ThirstForScience',
4696             'channel_follower_count': int
4697         }
4698     }, {
4699         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4700         'only_matching': True,
4701     }, {
4702         'note': 'basic, single video playlist',
4703         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4704         'info_dict': {
4705             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4706             'uploader': 'Sergey M.',
4707             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4708             'title': 'youtube-dl public playlist',
4709             'description': '',
4710             'tags': [],
4711             'view_count': int,
4712             'modified_date': '20201130',
4713             'channel': 'Sergey M.',
4714             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4715             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4716             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4717         },
4718         'playlist_count': 1,
4719     }, {
4720         'note': 'empty playlist',
4721         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4722         'info_dict': {
4723             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4724             'uploader': 'Sergey M.',
4725             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4726             'title': 'youtube-dl empty playlist',
4727             'tags': [],
4728             'channel': 'Sergey M.',
4729             'description': '',
4730             'modified_date': '20160902',
4731             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4732             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4733             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4734         },
4735         'playlist_count': 0,
4736     }, {
4737         'note': 'Home tab',
4738         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4739         'info_dict': {
4740             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4741             'title': 'lex will - Home',
4742             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4743             'uploader': 'lex will',
4744             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4745             'channel': 'lex will',
4746             'tags': ['bible', 'history', 'prophesy'],
4747             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4748             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4749             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4750             'channel_follower_count': int
4751         },
4752         'playlist_mincount': 2,
4753     }, {
4754         'note': 'Videos tab',
4755         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4756         'info_dict': {
4757             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4758             'title': 'lex will - Videos',
4759             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4760             'uploader': 'lex will',
4761             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4762             'tags': ['bible', 'history', 'prophesy'],
4763             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4764             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4765             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4766             'channel': 'lex will',
4767             'channel_follower_count': int
4768         },
4769         'playlist_mincount': 975,
4770     }, {
4771         'note': 'Videos tab, sorted by popular',
4772         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4773         'info_dict': {
4774             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4775             'title': 'lex will - Videos',
4776             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4777             'uploader': 'lex will',
4778             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4779             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4780             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4781             'channel': 'lex will',
4782             'tags': ['bible', 'history', 'prophesy'],
4783             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4784             'channel_follower_count': int
4785         },
4786         'playlist_mincount': 199,
4787     }, {
4788         'note': 'Playlists tab',
4789         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4790         'info_dict': {
4791             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4792             'title': 'lex will - Playlists',
4793             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4794             'uploader': 'lex will',
4795             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4796             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4797             'channel': 'lex will',
4798             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4799             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4800             'tags': ['bible', 'history', 'prophesy'],
4801             'channel_follower_count': int
4802         },
4803         'playlist_mincount': 17,
4804     }, {
4805         'note': 'Community tab',
4806         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4807         'info_dict': {
4808             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4809             'title': 'lex will - Community',
4810             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4811             'uploader': 'lex will',
4812             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4813             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4814             'channel': 'lex will',
4815             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4816             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4817             'tags': ['bible', 'history', 'prophesy'],
4818             'channel_follower_count': int
4819         },
4820         'playlist_mincount': 18,
4821     }, {
4822         'note': 'Channels tab',
4823         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4824         'info_dict': {
4825             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4826             'title': 'lex will - Channels',
4827             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4828             'uploader': 'lex will',
4829             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4830             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4831             'channel': 'lex will',
4832             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4833             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4834             'tags': ['bible', 'history', 'prophesy'],
4835             'channel_follower_count': int
4836         },
4837         'playlist_mincount': 12,
4838     }, {
4839         'note': 'Search tab',
4840         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4841         'playlist_mincount': 40,
4842         'info_dict': {
4843             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4844             'title': '3Blue1Brown - Search - linear algebra',
4845             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4846             'uploader': '3Blue1Brown',
4847             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4848             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4849             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4850             'tags': ['Mathematics'],
4851             'channel': '3Blue1Brown',
4852             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4853             'channel_follower_count': int
4854         },
4855     }, {
4856         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4857         'only_matching': True,
4858     }, {
4859         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4860         'only_matching': True,
4861     }, {
4862         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4863         'only_matching': True,
4864     }, {
4865         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4866         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4867         'info_dict': {
4868             'title': '29C3: Not my department',
4869             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4870             'uploader': 'Christiaan008',
4871             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4872             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4873             'tags': [],
4874             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4875             'view_count': int,
4876             'modified_date': '20150605',
4877             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4878             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4879             'channel': 'Christiaan008',
4880         },
4881         'playlist_count': 96,
4882     }, {
4883         'note': 'Large playlist',
4884         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4885         'info_dict': {
4886             'title': 'Uploads from Cauchemar',
4887             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4888             'uploader': 'Cauchemar',
4889             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4890             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4891             'tags': [],
4892             'modified_date': r're:\d{8}',
4893             'channel': 'Cauchemar',
4894             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4895             'view_count': int,
4896             'description': '',
4897             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4898         },
4899         'playlist_mincount': 1123,
4900         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4901     }, {
4902         'note': 'even larger playlist, 8832 videos',
4903         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4904         'only_matching': True,
4905     }, {
4906         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4907         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4908         'info_dict': {
4909             'title': 'Uploads from Interstellar Movie',
4910             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4911             'uploader': 'Interstellar Movie',
4912             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4913             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4914             'tags': [],
4915             'view_count': int,
4916             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4917             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4918             'channel': 'Interstellar Movie',
4919             'description': '',
4920             'modified_date': r're:\d{8}',
4921         },
4922         'playlist_mincount': 21,
4923     }, {
4924         'note': 'Playlist with "show unavailable videos" button',
4925         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4926         'info_dict': {
4927             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4928             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4929             'uploader': 'Phim Siêu Nhân Nhật Bản',
4930             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4931             'view_count': int,
4932             'channel': 'Phim Siêu Nhân Nhật Bản',
4933             'tags': [],
4934             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4935             'description': '',
4936             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4937             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4938             'modified_date': r're:\d{8}',
4939         },
4940         'playlist_mincount': 200,
4941         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4942     }, {
4943         'note': 'Playlist with unavailable videos in page 7',
4944         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4945         'info_dict': {
4946             'title': 'Uploads from BlankTV',
4947             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4948             'uploader': 'BlankTV',
4949             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4950             'channel': 'BlankTV',
4951             'channel_url': 'https://www.youtube.com/c/blanktv',
4952             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4953             'view_count': int,
4954             'tags': [],
4955             'uploader_url': 'https://www.youtube.com/c/blanktv',
4956             'modified_date': r're:\d{8}',
4957             'description': '',
4958         },
4959         'playlist_mincount': 1000,
4960         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4961     }, {
4962         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4963         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4964         'info_dict': {
4965             'title': 'Data Analysis with Dr Mike Pound',
4966             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4967             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4968             'uploader': 'Computerphile',
4969             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4970             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4971             'tags': [],
4972             'view_count': int,
4973             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4974             'channel_url': 'https://www.youtube.com/user/Computerphile',
4975             'channel': 'Computerphile',
4976         },
4977         'playlist_mincount': 11,
4978     }, {
4979         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4980         'only_matching': True,
4981     }, {
4982         'note': 'Playlist URL that does not actually serve a playlist',
4983         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4984         'info_dict': {
4985             'id': 'FqZTN594JQw',
4986             'ext': 'webm',
4987             'title': "Smiley's People 01 detective, Adventure Series, Action",
4988             'uploader': 'STREEM',
4989             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4990             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4991             'upload_date': '20150526',
4992             'license': 'Standard YouTube License',
4993             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4994             'categories': ['People & Blogs'],
4995             'tags': list,
4996             'view_count': int,
4997             'like_count': int,
4998         },
4999         'params': {
5000             'skip_download': True,
5001         },
5002         'skip': 'This video is not available.',
5003         'add_ie': [YoutubeIE.ie_key()],
5004     }, {
5005         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5006         'only_matching': True,
5007     }, {
5008         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5009         'only_matching': True,
5010     }, {
5011         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5012         'info_dict': {
5013             'id': 'GgL890LIznQ',  # This will keep changing
5014             'ext': 'mp4',
5015             'title': str,
5016             'uploader': 'Sky News',
5017             'uploader_id': 'skynews',
5018             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5019             'upload_date': r're:\d{8}',
5020             'description': str,
5021             'categories': ['News & Politics'],
5022             'tags': list,
5023             'like_count': int,
5024             'release_timestamp': 1642502819,
5025             'channel': 'Sky News',
5026             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5027             'age_limit': 0,
5028             'view_count': int,
5029             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5030             'playable_in_embed': True,
5031             'release_date': '20220118',
5032             'availability': 'public',
5033             'live_status': 'is_live',
5034             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5035             'channel_follower_count': int
5036         },
5037         'params': {
5038             'skip_download': True,
5039         },
5040         'expected_warnings': ['Ignoring subtitle tracks found in '],
5041     }, {
5042         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5043         'info_dict': {
5044             'id': 'a48o2S1cPoo',
5045             'ext': 'mp4',
5046             'title': 'The Young Turks - Live Main Show',
5047             'uploader': 'The Young Turks',
5048             'uploader_id': 'TheYoungTurks',
5049             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5050             'upload_date': '20150715',
5051             'license': 'Standard YouTube License',
5052             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5053             'categories': ['News & Politics'],
5054             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5055             'like_count': int,
5056         },
5057         'params': {
5058             'skip_download': True,
5059         },
5060         'only_matching': True,
5061     }, {
5062         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5063         'only_matching': True,
5064     }, {
5065         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5066         'only_matching': True,
5067     }, {
5068         'note': 'A channel that is not live. Should raise error',
5069         'url': 'https://www.youtube.com/user/numberphile/live',
5070         'only_matching': True,
5071     }, {
5072         'url': 'https://www.youtube.com/feed/trending',
5073         'only_matching': True,
5074     }, {
5075         'url': 'https://www.youtube.com/feed/library',
5076         'only_matching': True,
5077     }, {
5078         'url': 'https://www.youtube.com/feed/history',
5079         'only_matching': True,
5080     }, {
5081         'url': 'https://www.youtube.com/feed/subscriptions',
5082         'only_matching': True,
5083     }, {
5084         'url': 'https://www.youtube.com/feed/watch_later',
5085         'only_matching': True,
5086     }, {
5087         'note': 'Recommended - redirects to home page.',
5088         'url': 'https://www.youtube.com/feed/recommended',
5089         'only_matching': True,
5090     }, {
5091         'note': 'inline playlist with not always working continuations',
5092         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5093         'only_matching': True,
5094     }, {
5095         'url': 'https://www.youtube.com/course',
5096         'only_matching': True,
5097     }, {
5098         'url': 'https://www.youtube.com/zsecurity',
5099         'only_matching': True,
5100     }, {
5101         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5102         'only_matching': True,
5103     }, {
5104         'url': 'https://www.youtube.com/TheYoungTurks/live',
5105         'only_matching': True,
5106     }, {
5107         'url': 'https://www.youtube.com/hashtag/cctv9',
5108         'info_dict': {
5109             'id': 'cctv9',
5110             'title': '#cctv9',
5111             'tags': [],
5112         },
5113         'playlist_mincount': 350,
5114     }, {
5115         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5116         'only_matching': True,
5117     }, {
5118         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5119         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5120         'only_matching': True
5121     }, {
5122         'note': '/browse/ should redirect to /channel/',
5123         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5124         'only_matching': True
5125     }, {
5126         'note': 'VLPL, should redirect to playlist?list=PL...',
5127         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5128         'info_dict': {
5129             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5130             'uploader': 'NoCopyrightSounds',
5131             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5132             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5133             'title': 'NCS Releases',
5134             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5135             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5136             'modified_date': r're:\d{8}',
5137             'view_count': int,
5138             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5139             'tags': [],
5140             'channel': 'NoCopyrightSounds',
5141         },
5142         'playlist_mincount': 166,
5143         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5144     }, {
5145         'note': 'Topic, should redirect to playlist?list=UU...',
5146         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5147         'info_dict': {
5148             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5149             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5150             'title': 'Uploads from Royalty Free Music - Topic',
5151             'uploader': 'Royalty Free Music - Topic',
5152             'tags': [],
5153             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5154             'channel': 'Royalty Free Music - Topic',
5155             'view_count': int,
5156             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5157             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5158             'modified_date': r're:\d{8}',
5159             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5160             'description': '',
5161         },
5162         'expected_warnings': [
5163             'The URL does not have a videos tab',
5164             r'[Uu]navailable videos (are|will be) hidden',
5165         ],
5166         'playlist_mincount': 101,
5167     }, {
5168         'note': 'Topic without a UU playlist',
5169         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5170         'info_dict': {
5171             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5172             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5173             'tags': [],
5174         },
5175         'expected_warnings': [
5176             'the playlist redirect gave error',
5177         ],
5178         'playlist_mincount': 9,
5179     }, {
5180         'note': 'Youtube music Album',
5181         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5182         'info_dict': {
5183             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5184             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5185             'tags': [],
5186             'view_count': int,
5187             'description': '',
5188             'availability': 'unlisted',
5189             'modified_date': r're:\d{8}',
5190         },
5191         'playlist_count': 50,
5192     }, {
5193         'note': 'unlisted single video playlist',
5194         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5195         'info_dict': {
5196             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5197             'uploader': 'colethedj',
5198             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5199             'title': 'yt-dlp unlisted playlist test',
5200             'availability': 'unlisted',
5201             'tags': [],
5202             'modified_date': '20211208',
5203             'channel': 'colethedj',
5204             'view_count': int,
5205             'description': '',
5206             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5207             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5208             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5209         },
5210         'playlist_count': 1,
5211     }, {
5212         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5213         'url': 'https://www.youtube.com/feed/recommended',
5214         'info_dict': {
5215             'id': 'recommended',
5216             'title': 'recommended',
5217             'tags': [],
5218         },
5219         'playlist_mincount': 50,
5220         'params': {
5221             'skip_download': True,
5222             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5223         },
5224     }, {
5225         'note': 'API Fallback: /videos tab, sorted by oldest first',
5226         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5227         'info_dict': {
5228             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5229             'title': 'Cody\'sLab - Videos',
5230             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5231             'uploader': 'Cody\'sLab',
5232             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5233             'channel': 'Cody\'sLab',
5234             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5235             'tags': [],
5236             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5237             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5238             'channel_follower_count': int
5239         },
5240         'playlist_mincount': 650,
5241         'params': {
5242             'skip_download': True,
5243             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5244         },
5245     }, {
5246         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5247         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5248         'info_dict': {
5249             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5250             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5251             'title': 'Uploads from Royalty Free Music - Topic',
5252             'uploader': 'Royalty Free Music - Topic',
5253             'modified_date': r're:\d{8}',
5254             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5255             'description': '',
5256             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5257             'tags': [],
5258             'channel': 'Royalty Free Music - Topic',
5259             'view_count': int,
5260             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5261         },
5262         'expected_warnings': [
5263             'does not have a videos tab',
5264             r'[Uu]navailable videos (are|will be) hidden',
5265         ],
5266         'playlist_mincount': 101,
5267         'params': {
5268             'skip_download': True,
5269             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5270         },
5271     }, {
5272         'note': 'non-standard redirect to regional channel',
5273         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5274         'only_matching': True
5275     }, {
5276         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5277         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5278         'info_dict': {
5279             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5280             'modified_date': '20220407',
5281             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5282             'tags': [],
5283             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5284             'uploader': 'pukkandan',
5285             'availability': 'unlisted',
5286             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5287             'channel': 'pukkandan',
5288             'description': 'Test for collaborative playlist',
5289             'title': 'yt-dlp test - collaborative playlist',
5290             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5291         },
5292         'playlist_mincount': 2
5293     }]
5294
5295     @classmethod
5296     def suitable(cls, url):
5297         return False if YoutubeIE.suitable(url) else super().suitable(url)
5298
5299     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5300
5301     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5302     def _real_extract(self, url, smuggled_data):
5303         item_id = self._match_id(url)
5304         url = compat_urlparse.urlunparse(
5305             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
5306         compat_opts = self.get_param('compat_opts', [])
5307
5308         def get_mobj(url):
5309             mobj = self._URL_RE.match(url).groupdict()
5310             mobj.update((k, '') for k, v in mobj.items() if v is None)
5311             return mobj
5312
5313         mobj, redirect_warning = get_mobj(url), None
5314         # Youtube returns incomplete data if tabname is not lower case
5315         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5316         if is_channel:
5317             if smuggled_data.get('is_music_url'):
5318                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5319                     item_id = item_id[2:]
5320                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5321                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5322                     mdata = self._extract_tab_endpoint(
5323                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5324                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5325                                         get_all=False, expected_type=compat_str)
5326                     if not murl:
5327                         raise ExtractorError('Failed to resolve album to playlist')
5328                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5329                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5330                     pre = f'https://www.youtube.com/channel/{item_id}'
5331
5332         original_tab_name = tab
5333         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5334             # Home URLs should redirect to /videos/
5335             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5336                                 'To download only the videos in the home page, add a "/featured" to the URL')
5337             tab = '/videos'
5338
5339         url = ''.join((pre, tab, post))
5340         mobj = get_mobj(url)
5341
5342         # Handle both video/playlist URLs
5343         qs = parse_qs(url)
5344         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5345
5346         if not video_id and mobj['not_channel'].startswith('watch'):
5347             if not playlist_id:
5348                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5349                 raise ExtractorError('Unable to recognize tab page')
5350             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5351             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5352             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5353             mobj = get_mobj(url)
5354
5355         if video_id and playlist_id:
5356             if self.get_param('noplaylist'):
5357                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5358                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5359                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5360             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5361
5362         data, ytcfg = self._extract_data(url, item_id)
5363
5364         # YouTube may provide a non-standard redirect to the regional channel
5365         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5366         redirect_url = traverse_obj(
5367             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5368         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5369             redirect_url = ''.join((
5370                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5371             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5372             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5373
5374         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5375         if tabs:
5376             selected_tab = self._extract_selected_tab(tabs)
5377             selected_tab_name = selected_tab.get('title', '').lower()
5378             if selected_tab_name == 'home':
5379                 selected_tab_name = 'featured'
5380             requested_tab_name = mobj['tab'][1:]
5381             if 'no-youtube-channel-redirect' not in compat_opts:
5382                 if requested_tab_name == 'live':
5383                     # Live tab should have redirected to the video
5384                     raise ExtractorError('The channel is not currently live', expected=True)
5385                 if requested_tab_name not in ('', selected_tab_name):
5386                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5387                     if not original_tab_name:
5388                         if item_id[:2] == 'UC':
5389                             # Topic channels don't have /videos. Use the equivalent playlist instead
5390                             pl_id = f'UU{item_id[2:]}'
5391                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5392                             try:
5393                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5394                             except ExtractorError:
5395                                 redirect_warning += ' and the playlist redirect gave error'
5396                             else:
5397                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5398                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5399                         if selected_tab_name and selected_tab_name != requested_tab_name:
5400                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5401                     else:
5402                         raise ExtractorError(redirect_warning, expected=True)
5403
5404         if redirect_warning:
5405             self.to_screen(redirect_warning)
5406         self.write_debug(f'Final URL: {url}')
5407
5408         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5409         if 'no-youtube-unavailable-videos' not in compat_opts:
5410             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5411         self._extract_and_report_alerts(data, only_once=True)
5412         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5413         if tabs:
5414             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5415
5416         playlist = traverse_obj(
5417             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5418         if playlist:
5419             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5420
5421         video_id = traverse_obj(
5422             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5423         if video_id:
5424             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5425                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5426             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5427                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5428
5429         raise ExtractorError('Unable to recognize tab page')
5430
5431
5432 class YoutubePlaylistIE(InfoExtractor):
5433     IE_DESC = 'YouTube playlists'
5434     _VALID_URL = r'''(?x)(?:
5435                         (?:https?://)?
5436                         (?:\w+\.)?
5437                         (?:
5438                             (?:
5439                                 youtube(?:kids)?\.com|
5440                                 %(invidious)s
5441                             )
5442                             /.*?\?.*?\blist=
5443                         )?
5444                         (?P<id>%(playlist_id)s)
5445                      )''' % {
5446         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5447         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5448     }
5449     IE_NAME = 'youtube:playlist'
5450     _TESTS = [{
5451         'note': 'issue #673',
5452         'url': 'PLBB231211A4F62143',
5453         'info_dict': {
5454             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5455             'id': 'PLBB231211A4F62143',
5456             'uploader': 'Wickman',
5457             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5458             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5459             'view_count': int,
5460             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5461             'modified_date': r're:\d{8}',
5462             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5463             'channel': 'Wickman',
5464             'tags': [],
5465             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5466         },
5467         'playlist_mincount': 29,
5468     }, {
5469         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5470         'info_dict': {
5471             'title': 'YDL_safe_search',
5472             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5473         },
5474         'playlist_count': 2,
5475         'skip': 'This playlist is private',
5476     }, {
5477         'note': 'embedded',
5478         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5479         'playlist_count': 4,
5480         'info_dict': {
5481             'title': 'JODA15',
5482             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5483             'uploader': 'milan',
5484             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5485             'description': '',
5486             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5487             'tags': [],
5488             'modified_date': '20140919',
5489             'view_count': int,
5490             'channel': 'milan',
5491             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5492             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5493         },
5494         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5495     }, {
5496         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5497         'playlist_mincount': 654,
5498         'info_dict': {
5499             'title': '2018 Chinese New Singles (11/6 updated)',
5500             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5501             'uploader': 'LBK',
5502             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5503             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5504             'channel': 'LBK',
5505             'view_count': int,
5506             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5507             'tags': [],
5508             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5509             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5510             'modified_date': r're:\d{8}',
5511         },
5512         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5513     }, {
5514         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5515         'only_matching': True,
5516     }, {
5517         # music album playlist
5518         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5519         'only_matching': True,
5520     }]
5521
5522     @classmethod
5523     def suitable(cls, url):
5524         if YoutubeTabIE.suitable(url):
5525             return False
5526         from ..utils import parse_qs
5527         qs = parse_qs(url)
5528         if qs.get('v', [None])[0]:
5529             return False
5530         return super().suitable(url)
5531
5532     def _real_extract(self, url):
5533         playlist_id = self._match_id(url)
5534         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5535         url = update_url_query(
5536             'https://www.youtube.com/playlist',
5537             parse_qs(url) or {'list': playlist_id})
5538         if is_music_url:
5539             url = smuggle_url(url, {'is_music_url': True})
5540         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5541
5542
5543 class YoutubeYtBeIE(InfoExtractor):
5544     IE_DESC = 'youtu.be'
5545     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5546     _TESTS = [{
5547         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5548         'info_dict': {
5549             'id': 'yeWKywCrFtk',
5550             'ext': 'mp4',
5551             'title': 'Small Scale Baler and Braiding Rugs',
5552             'uploader': 'Backus-Page House Museum',
5553             'uploader_id': 'backuspagemuseum',
5554             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5555             'upload_date': '20161008',
5556             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5557             'categories': ['Nonprofits & Activism'],
5558             'tags': list,
5559             'like_count': int,
5560             'age_limit': 0,
5561             'playable_in_embed': True,
5562             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5563             'channel': 'Backus-Page House Museum',
5564             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5565             'live_status': 'not_live',
5566             'view_count': int,
5567             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5568             'availability': 'public',
5569             'duration': 59,
5570         },
5571         'params': {
5572             'noplaylist': True,
5573             'skip_download': True,
5574         },
5575     }, {
5576         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5577         'only_matching': True,
5578     }]
5579
5580     def _real_extract(self, url):
5581         mobj = self._match_valid_url(url)
5582         video_id = mobj.group('id')
5583         playlist_id = mobj.group('playlist_id')
5584         return self.url_result(
5585             update_url_query('https://www.youtube.com/watch', {
5586                 'v': video_id,
5587                 'list': playlist_id,
5588                 'feature': 'youtu.be',
5589             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5590
5591
5592 class YoutubeLivestreamEmbedIE(InfoExtractor):
5593     IE_DESC = 'YouTube livestream embeds'
5594     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5595     _TESTS = [{
5596         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5597         'only_matching': True,
5598     }]
5599
5600     def _real_extract(self, url):
5601         channel_id = self._match_id(url)
5602         return self.url_result(
5603             f'https://www.youtube.com/channel/{channel_id}/live',
5604             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5605
5606
5607 class YoutubeYtUserIE(InfoExtractor):
5608     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5609     IE_NAME = 'youtube:user'
5610     _VALID_URL = r'ytuser:(?P<id>.+)'
5611     _TESTS = [{
5612         'url': 'ytuser:phihag',
5613         'only_matching': True,
5614     }]
5615
5616     def _real_extract(self, url):
5617         user_id = self._match_id(url)
5618         return self.url_result(
5619             'https://www.youtube.com/user/%s/videos' % user_id,
5620             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5621
5622
5623 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5624     IE_NAME = 'youtube:favorites'
5625     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5626     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5627     _LOGIN_REQUIRED = True
5628     _TESTS = [{
5629         'url': ':ytfav',
5630         'only_matching': True,
5631     }, {
5632         'url': ':ytfavorites',
5633         'only_matching': True,
5634     }]
5635
5636     def _real_extract(self, url):
5637         return self.url_result(
5638             'https://www.youtube.com/playlist?list=LL',
5639             ie=YoutubeTabIE.ie_key())
5640
5641
5642 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5643     IE_NAME = 'youtube:notif'
5644     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5645     _VALID_URL = r':ytnotif(?:ication)?s?'
5646     _LOGIN_REQUIRED = True
5647     _TESTS = [{
5648         'url': ':ytnotif',
5649         'only_matching': True,
5650     }, {
5651         'url': ':ytnotifications',
5652         'only_matching': True,
5653     }]
5654
5655     def _extract_notification_menu(self, response, continuation_list):
5656         notification_list = traverse_obj(
5657             response,
5658             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5659             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5660             expected_type=list) or []
5661         continuation_list[0] = None
5662         for item in notification_list:
5663             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5664             if entry:
5665                 yield entry
5666             continuation = item.get('continuationItemRenderer')
5667             if continuation:
5668                 continuation_list[0] = continuation
5669
5670     def _extract_notification_renderer(self, notification):
5671         video_id = traverse_obj(
5672             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5673         url = f'https://www.youtube.com/watch?v={video_id}'
5674         channel_id = None
5675         if not video_id:
5676             browse_ep = traverse_obj(
5677                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5678             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5679             post_id = self._search_regex(
5680                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5681                 'post id', default=None)
5682             if not channel_id or not post_id:
5683                 return
5684             # The direct /post url redirects to this in the browser
5685             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5686
5687         channel = traverse_obj(
5688             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5689             expected_type=str)
5690         notification_title = self._get_text(notification, 'shortMessage')
5691         if notification_title:
5692             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
5693         # TODO: handle recommended videos
5694         title = self._search_regex(
5695             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
5696             'video title', default=None)
5697         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5698                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5699                        else None)
5700         return {
5701             '_type': 'url',
5702             'url': url,
5703             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5704             'video_id': video_id,
5705             'title': title,
5706             'channel_id': channel_id,
5707             'channel': channel,
5708             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5709             'upload_date': upload_date,
5710         }
5711
5712     def _notification_menu_entries(self, ytcfg):
5713         continuation_list = [None]
5714         response = None
5715         for page in itertools.count(1):
5716             ctoken = traverse_obj(
5717                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5718             response = self._extract_response(
5719                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5720                 ep='notification/get_notification_menu', check_get_keys='actions',
5721                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5722             yield from self._extract_notification_menu(response, continuation_list)
5723             if not continuation_list[0]:
5724                 break
5725
5726     def _real_extract(self, url):
5727         display_id = 'notifications'
5728         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5729         self._report_playlist_authcheck(ytcfg)
5730         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5731
5732
5733 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5734     IE_DESC = 'YouTube search'
5735     IE_NAME = 'youtube:search'
5736     _SEARCH_KEY = 'ytsearch'
5737     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5738     _TESTS = [{
5739         'url': 'ytsearch5:youtube-dl test video',
5740         'playlist_count': 5,
5741         'info_dict': {
5742             'id': 'youtube-dl test video',
5743             'title': 'youtube-dl test video',
5744         }
5745     }]
5746
5747
5748 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5749     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5750     _SEARCH_KEY = 'ytsearchdate'
5751     IE_DESC = 'YouTube search, newest videos first'
5752     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5753     _TESTS = [{
5754         'url': 'ytsearchdate5:youtube-dl test video',
5755         'playlist_count': 5,
5756         'info_dict': {
5757             'id': 'youtube-dl test video',
5758             'title': 'youtube-dl test video',
5759         }
5760     }]
5761
5762
5763 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5764     IE_DESC = 'YouTube search URLs with sorting and filter support'
5765     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5766     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5767     _TESTS = [{
5768         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5769         'playlist_mincount': 5,
5770         'info_dict': {
5771             'id': 'youtube-dl test video',
5772             'title': 'youtube-dl test video',
5773         }
5774     }, {
5775         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5776         'playlist_mincount': 5,
5777         'info_dict': {
5778             'id': 'python',
5779             'title': 'python',
5780         }
5781     }, {
5782         'url': 'https://www.youtube.com/results?search_query=%23cats',
5783         'playlist_mincount': 1,
5784         'info_dict': {
5785             'id': '#cats',
5786             'title': '#cats',
5787             'entries': [{
5788                 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5789                 'title': '#cats',
5790             }],
5791         },
5792     }, {
5793         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5794         'only_matching': True,
5795     }]
5796
5797     def _real_extract(self, url):
5798         qs = parse_qs(url)
5799         query = (qs.get('search_query') or qs.get('q'))[0]
5800         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5801
5802
5803 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5804     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5805     IE_NAME = 'youtube:music:search_url'
5806     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5807     _TESTS = [{
5808         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5809         'playlist_count': 16,
5810         'info_dict': {
5811             'id': 'royalty free music',
5812             'title': 'royalty free music',
5813         }
5814     }, {
5815         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5816         'playlist_mincount': 30,
5817         'info_dict': {
5818             'id': 'royalty free music - songs',
5819             'title': 'royalty free music - songs',
5820         },
5821         'params': {'extract_flat': 'in_playlist'}
5822     }, {
5823         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5824         'playlist_mincount': 30,
5825         'info_dict': {
5826             'id': 'royalty free music - community playlists',
5827             'title': 'royalty free music - community playlists',
5828         },
5829         'params': {'extract_flat': 'in_playlist'}
5830     }]
5831
5832     _SECTIONS = {
5833         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5834         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5835         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5836         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5837         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5838         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5839     }
5840
5841     def _real_extract(self, url):
5842         qs = parse_qs(url)
5843         query = (qs.get('search_query') or qs.get('q'))[0]
5844         params = qs.get('sp', (None,))[0]
5845         if params:
5846             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5847         else:
5848             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
5849             params = self._SECTIONS.get(section)
5850             if not params:
5851                 section = None
5852         title = join_nonempty(query, section, delim=' - ')
5853         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5854
5855
5856 class YoutubeFeedsInfoExtractor(InfoExtractor):
5857     """
5858     Base class for feed extractors
5859     Subclasses must re-define the _FEED_NAME property.
5860     """
5861     _LOGIN_REQUIRED = True
5862     _FEED_NAME = 'feeds'
5863
5864     def _real_initialize(self):
5865         YoutubeBaseInfoExtractor._check_login_required(self)
5866
5867     @classproperty
5868     def IE_NAME(self):
5869         return f'youtube:{self._FEED_NAME}'
5870
5871     def _real_extract(self, url):
5872         return self.url_result(
5873             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5874
5875
5876 class YoutubeWatchLaterIE(InfoExtractor):
5877     IE_NAME = 'youtube:watchlater'
5878     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5879     _VALID_URL = r':ytwatchlater'
5880     _TESTS = [{
5881         'url': ':ytwatchlater',
5882         'only_matching': True,
5883     }]
5884
5885     def _real_extract(self, url):
5886         return self.url_result(
5887             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5888
5889
5890 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5891     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5892     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5893     _FEED_NAME = 'recommended'
5894     _LOGIN_REQUIRED = False
5895     _TESTS = [{
5896         'url': ':ytrec',
5897         'only_matching': True,
5898     }, {
5899         'url': ':ytrecommended',
5900         'only_matching': True,
5901     }, {
5902         'url': 'https://youtube.com',
5903         'only_matching': True,
5904     }]
5905
5906
5907 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5908     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5909     _VALID_URL = r':ytsub(?:scription)?s?'
5910     _FEED_NAME = 'subscriptions'
5911     _TESTS = [{
5912         'url': ':ytsubs',
5913         'only_matching': True,
5914     }, {
5915         'url': ':ytsubscriptions',
5916         'only_matching': True,
5917     }]
5918
5919
5920 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5921     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5922     _VALID_URL = r':ythis(?:tory)?'
5923     _FEED_NAME = 'history'
5924     _TESTS = [{
5925         'url': ':ythistory',
5926         'only_matching': True,
5927     }]
5928
5929
5930 class YoutubeStoriesIE(InfoExtractor):
5931     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5932     IE_NAME = 'youtube:stories'
5933     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5934     _TESTS = [{
5935         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5936         'only_matching': True,
5937     }]
5938
5939     def _real_extract(self, url):
5940         playlist_id = f'RLTD{self._match_id(url)}'
5941         return self.url_result(
5942             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5943             ie=YoutubeTabIE, video_id=playlist_id)
5944
5945
5946 class YoutubeTruncatedURLIE(InfoExtractor):
5947     IE_NAME = 'youtube:truncated_url'
5948     IE_DESC = False  # Do not list
5949     _VALID_URL = r'''(?x)
5950         (?:https?://)?
5951         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5952         (?:watch\?(?:
5953             feature=[a-z_]+|
5954             annotation_id=annotation_[^&]+|
5955             x-yt-cl=[0-9]+|
5956             hl=[^&]*|
5957             t=[0-9]+
5958         )?
5959         |
5960             attribution_link\?a=[^&]+
5961         )
5962         $
5963     '''
5964
5965     _TESTS = [{
5966         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5967         'only_matching': True,
5968     }, {
5969         'url': 'https://www.youtube.com/watch?',
5970         'only_matching': True,
5971     }, {
5972         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5973         'only_matching': True,
5974     }, {
5975         'url': 'https://www.youtube.com/watch?feature=foo',
5976         'only_matching': True,
5977     }, {
5978         'url': 'https://www.youtube.com/watch?hl=en-GB',
5979         'only_matching': True,
5980     }, {
5981         'url': 'https://www.youtube.com/watch?t=2372',
5982         'only_matching': True,
5983     }]
5984
5985     def _real_extract(self, url):
5986         raise ExtractorError(
5987             'Did you forget to quote the URL? Remember that & is a meta '
5988             'character in most shells, so you want to put the URL in quotes, '
5989             'like  youtube-dl '
5990             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5991             ' or simply  youtube-dl BaW_jenozKc  .',
5992             expected=True)
5993
5994
5995 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
5996     IE_NAME = 'youtube:clip'
5997     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
5998     _TESTS = [{
5999         # FIXME: Other metadata should be extracted from the clip, not from the base video
6000         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6001         'info_dict': {
6002             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6003             'ext': 'mp4',
6004             'section_start': 29.0,
6005             'section_end': 39.7,
6006             'duration': 10.7,
6007         }
6008     }]
6009
6010     def _real_extract(self, url):
6011         clip_id = self._match_id(url)
6012         _, data = self._extract_webpage(url, clip_id)
6013
6014         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6015         if not video_id:
6016             raise ExtractorError('Unable to find video ID')
6017
6018         clip_data = traverse_obj(data, (
6019             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6020             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6021             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6022             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6023
6024         return {
6025             '_type': 'url_transparent',
6026             'url': f'https://www.youtube.com/watch?v={video_id}',
6027             'ie_key': YoutubeIE.ie_key(),
6028             'id': clip_id,
6029             'section_start': int(clip_data['startTimeMs']) / 1000,
6030             'section_end': int(clip_data['endTimeMs']) / 1000,
6031         }
6032
6033
6034 class YoutubeTruncatedIDIE(InfoExtractor):
6035     IE_NAME = 'youtube:truncated_id'
6036     IE_DESC = False  # Do not list
6037     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6038
6039     _TESTS = [{
6040         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6041         'only_matching': True,
6042     }]
6043
6044     def _real_extract(self, url):
6045         video_id = self._match_id(url)
6046         raise ExtractorError(
6047             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6048             expected=True)