yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16 import urllib.error
  17 import urllib.parse
  18
  19 from .common import InfoExtractor, SearchInfoExtractor
  20 from ..compat import functools
  21 from ..jsinterp import JSInterpreter
  22 from ..utils import (
  23     NO_DEFAULT,
  24     ExtractorError,
  25     bug_reports_message,
  26     classproperty,
  27     clean_html,
  28     datetime_from_str,
  29     dict_get,
  30     error_to_compat_str,
  31     float_or_none,
  32     format_field,
  33     get_first,
  34     int_or_none,
  35     is_html,
  36     join_nonempty,
  37     js_to_json,
  38     mimetype2ext,
  39     network_exceptions,
  40     orderedSet,
  41     parse_codecs,
  42     parse_count,
  43     parse_duration,
  44     parse_iso8601,
  45     parse_qs,
  46     qualities,
  47     remove_end,
  48     remove_start,
  49     smuggle_url,
  50     str_or_none,
  51     str_to_int,
  52     strftime_or_none,
  53     traverse_obj,
  54     try_get,
  55     unescapeHTML,
  56     unified_strdate,
  57     unified_timestamp,
  58     unsmuggle_url,
  59     update_url_query,
  60     url_or_none,
  61     urljoin,
  62     variadic,
  63 )
  64
  65 # any clients starting with _ cannot be explicitly requested by the user
  66 INNERTUBE_CLIENTS = {
  67     'web': {
  68         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  69         'INNERTUBE_CONTEXT': {
  70             'client': {
  71                 'clientName': 'WEB',
  72                 'clientVersion': '2.20211221.00.00',
  73             }
  74         },
  75         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  76     },
  77     'web_embedded': {
  78         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  79         'INNERTUBE_CONTEXT': {
  80             'client': {
  81                 'clientName': 'WEB_EMBEDDED_PLAYER',
  82                 'clientVersion': '1.20211215.00.01',
  83             },
  84         },
  85         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  86     },
  87     'web_music': {
  88         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  89         'INNERTUBE_HOST': 'music.youtube.com',
  90         'INNERTUBE_CONTEXT': {
  91             'client': {
  92                 'clientName': 'WEB_REMIX',
  93                 'clientVersion': '1.20211213.00.00',
  94             }
  95         },
  96         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
  97     },
  98     'web_creator': {
  99         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 100         'INNERTUBE_CONTEXT': {
 101             'client': {
 102                 'clientName': 'WEB_CREATOR',
 103                 'clientVersion': '1.20211220.02.00',
 104             }
 105         },
 106         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 107     },
 108     'android': {
 109         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 110         'INNERTUBE_CONTEXT': {
 111             'client': {
 112                 'clientName': 'ANDROID',
 113                 'clientVersion': '16.49',
 114             }
 115         },
 116         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 117         'REQUIRE_JS_PLAYER': False
 118     },
 119     'android_embedded': {
 120         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 121         'INNERTUBE_CONTEXT': {
 122             'client': {
 123                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 124                 'clientVersion': '16.49',
 125             },
 126         },
 127         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 128         'REQUIRE_JS_PLAYER': False
 129     },
 130     'android_music': {
 131         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 132         'INNERTUBE_CONTEXT': {
 133             'client': {
 134                 'clientName': 'ANDROID_MUSIC',
 135                 'clientVersion': '4.57',
 136             }
 137         },
 138         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 139         'REQUIRE_JS_PLAYER': False
 140     },
 141     'android_creator': {
 142         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 143         'INNERTUBE_CONTEXT': {
 144             'client': {
 145                 'clientName': 'ANDROID_CREATOR',
 146                 'clientVersion': '21.47',
 147             },
 148         },
 149         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 150         'REQUIRE_JS_PLAYER': False
 151     },
 152     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 153     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 154     'ios': {
 155         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 156         'INNERTUBE_CONTEXT': {
 157             'client': {
 158                 'clientName': 'IOS',
 159                 'clientVersion': '16.46',
 160                 'deviceModel': 'iPhone14,3',
 161             }
 162         },
 163         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 164         'REQUIRE_JS_PLAYER': False
 165     },
 166     'ios_embedded': {
 167         'INNERTUBE_CONTEXT': {
 168             'client': {
 169                 'clientName': 'IOS_MESSAGES_EXTENSION',
 170                 'clientVersion': '16.46',
 171                 'deviceModel': 'iPhone14,3',
 172             },
 173         },
 174         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 175         'REQUIRE_JS_PLAYER': False
 176     },
 177     'ios_music': {
 178         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 179         'INNERTUBE_CONTEXT': {
 180             'client': {
 181                 'clientName': 'IOS_MUSIC',
 182                 'clientVersion': '4.57',
 183             },
 184         },
 185         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 186         'REQUIRE_JS_PLAYER': False
 187     },
 188     'ios_creator': {
 189         'INNERTUBE_CONTEXT': {
 190             'client': {
 191                 'clientName': 'IOS_CREATOR',
 192                 'clientVersion': '21.47',
 193             },
 194         },
 195         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 196         'REQUIRE_JS_PLAYER': False
 197     },
 198     # mweb has 'ultralow' formats
 199     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 200     'mweb': {
 201         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 202         'INNERTUBE_CONTEXT': {
 203             'client': {
 204                 'clientName': 'MWEB',
 205                 'clientVersion': '2.20211221.01.00',
 206             }
 207         },
 208         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 209     },
 210     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 211     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 212     'tv_embedded': {
 213         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 214         'INNERTUBE_CONTEXT': {
 215             'client': {
 216                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 217                 'clientVersion': '2.0',
 218             },
 219         },
 220         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 221     },
 222 }
 223
 224
 225 def _split_innertube_client(client_name):
 226     variant, *base = client_name.rsplit('.', 1)
 227     if base:
 228         return variant, base[0], variant
 229     base, *variant = client_name.split('_', 1)
 230     return client_name, base, variant[0] if variant else None
 231
 232
 233 def build_innertube_clients():
 234     THIRD_PARTY = {
 235         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 236     }
 237     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 238     priority = qualities(BASE_CLIENTS[::-1])
 239
 240     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 241         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 242         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 243         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 244         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 245
 246         _, base_client, variant = _split_innertube_client(client)
 247         ytcfg['priority'] = 10 * priority(base_client)
 248
 249         if not variant:
 250             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 251             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 252             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 253             embedscreen['priority'] -= 3
 254         elif variant == 'embedded':
 255             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 256             ytcfg['priority'] -= 2
 257         else:
 258             ytcfg['priority'] -= 3
 259
 260
 261 build_innertube_clients()
 262
 263
 264 class YoutubeBaseInfoExtractor(InfoExtractor):
 265     """Provide base functions for Youtube extractors"""
 266
 267     _RESERVED_NAMES = (
 268         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 269         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 270         r'browse|oembed|get_video_info|iframe_api|s/player|'
 271         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 272
 273     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 274
 275     # _NETRC_MACHINE = 'youtube'
 276
 277     # If True it will raise an error if no login info is provided
 278     _LOGIN_REQUIRED = False
 279
 280     _INVIDIOUS_SITES = (
 281         # invidious-redirect websites
 282         r'(?:www\.)?redirect\.invidious\.io',
 283         r'(?:(?:www|dev)\.)?invidio\.us',
 284         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 285         r'(?:www\.)?invidious\.pussthecat\.org',
 286         r'(?:www\.)?invidious\.zee\.li',
 287         r'(?:www\.)?invidious\.ethibox\.fr',
 288         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 289         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 290         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 291         # youtube-dl invidious instances list
 292         r'(?:(?:www|no)\.)?invidiou\.sh',
 293         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 294         r'(?:www\.)?invidious\.kabi\.tk',
 295         r'(?:www\.)?invidious\.mastodon\.host',
 296         r'(?:www\.)?invidious\.zapashcanon\.fr',
 297         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 298         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 299         r'(?:www\.)?invidious\.himiko\.cloud',
 300         r'(?:www\.)?invidious\.reallyancient\.tech',
 301         r'(?:www\.)?invidious\.tube',
 302         r'(?:www\.)?invidiou\.site',
 303         r'(?:www\.)?invidious\.site',
 304         r'(?:www\.)?invidious\.xyz',
 305         r'(?:www\.)?invidious\.nixnet\.xyz',
 306         r'(?:www\.)?invidious\.048596\.xyz',
 307         r'(?:www\.)?invidious\.drycat\.fr',
 308         r'(?:www\.)?inv\.skyn3t\.in',
 309         r'(?:www\.)?tube\.poal\.co',
 310         r'(?:www\.)?tube\.connect\.cafe',
 311         r'(?:www\.)?vid\.wxzm\.sx',
 312         r'(?:www\.)?vid\.mint\.lgbt',
 313         r'(?:www\.)?vid\.puffyan\.us',
 314         r'(?:www\.)?yewtu\.be',
 315         r'(?:www\.)?yt\.elukerio\.org',
 316         r'(?:www\.)?yt\.lelux\.fi',
 317         r'(?:www\.)?invidious\.ggc-project\.de',
 318         r'(?:www\.)?yt\.maisputain\.ovh',
 319         r'(?:www\.)?ytprivate\.com',
 320         r'(?:www\.)?invidious\.13ad\.de',
 321         r'(?:www\.)?invidious\.toot\.koeln',
 322         r'(?:www\.)?invidious\.fdn\.fr',
 323         r'(?:www\.)?watch\.nettohikari\.com',
 324         r'(?:www\.)?invidious\.namazso\.eu',
 325         r'(?:www\.)?invidious\.silkky\.cloud',
 326         r'(?:www\.)?invidious\.exonip\.de',
 327         r'(?:www\.)?invidious\.riverside\.rocks',
 328         r'(?:www\.)?invidious\.blamefran\.net',
 329         r'(?:www\.)?invidious\.moomoo\.de',
 330         r'(?:www\.)?ytb\.trom\.tf',
 331         r'(?:www\.)?yt\.cyberhost\.uk',
 332         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 333         r'(?:www\.)?qklhadlycap4cnod\.onion',
 334         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 335         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 336         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 337         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 338         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 339         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 340         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 341         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 342         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 343         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 344         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 345         r'(?:www\.)?piped\.kavin\.rocks',
 346         r'(?:www\.)?piped\.silkky\.cloud',
 347         r'(?:www\.)?piped\.tokhmi\.xyz',
 348         r'(?:www\.)?piped\.moomoo\.me',
 349         r'(?:www\.)?il\.ax',
 350         r'(?:www\.)?piped\.syncpundit\.com',
 351         r'(?:www\.)?piped\.mha\.fi',
 352         r'(?:www\.)?piped\.mint\.lgbt',
 353         r'(?:www\.)?piped\.privacy\.com\.de',
 354     )
 355
 356     def _initialize_consent(self):
 357         cookies = self._get_cookies('https://www.youtube.com/')
 358         if cookies.get('__Secure-3PSID'):
 359             return
 360         consent_id = None
 361         consent = cookies.get('CONSENT')
 362         if consent:
 363             if 'YES' in consent.value:
 364                 return
 365             consent_id = self._search_regex(
 366                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 367         if not consent_id:
 368             consent_id = random.randint(100, 999)
 369         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 370
 371     def _initialize_pref(self):
 372         cookies = self._get_cookies('https://www.youtube.com/')
 373         pref_cookie = cookies.get('PREF')
 374         pref = {}
 375         if pref_cookie:
 376             try:
 377                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 378             except ValueError:
 379                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 380         pref.update({'hl': 'en', 'tz': 'UTC'})
 381         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 382
 383     def _real_initialize(self):
 384         self._initialize_pref()
 385         self._initialize_consent()
 386         self._check_login_required()
 387
 388     def _check_login_required(self):
 389         if self._LOGIN_REQUIRED and not self._cookies_passed:
 390             self.raise_login_required('Login details are needed to download this content', method='cookies')
 391
 392     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 393     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 394
 395     def _get_default_ytcfg(self, client='web'):
 396         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 397
 398     def _get_innertube_host(self, client='web'):
 399         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 400
 401     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 402         # try_get but with fallback to default ytcfg client values when present
 403         _func = lambda y: try_get(y, getter, expected_type)
 404         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 405
 406     def _extract_client_name(self, ytcfg, default_client='web'):
 407         return self._ytcfg_get_safe(
 408             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 409                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 410
 411     def _extract_client_version(self, ytcfg, default_client='web'):
 412         return self._ytcfg_get_safe(
 413             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 414                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 415
 416     def _select_api_hostname(self, req_api_hostname, default_client=None):
 417         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 418                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 419
 420     def _extract_api_key(self, ytcfg=None, default_client='web'):
 421         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 422
 423     def _extract_context(self, ytcfg=None, default_client='web'):
 424         context = get_first(
 425             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 426         # Enforce language and tz for extraction
 427         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 428         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 429         return context
 430
 431     _SAPISID = None
 432
 433     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 434         time_now = round(time.time())
 435         if self._SAPISID is None:
 436             yt_cookies = self._get_cookies('https://www.youtube.com')
 437             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 438             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 439             sapisid_cookie = dict_get(
 440                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 441             if sapisid_cookie and sapisid_cookie.value:
 442                 self._SAPISID = sapisid_cookie.value
 443                 self.write_debug('Extracted SAPISID cookie')
 444                 # SAPISID cookie is required if not already present
 445                 if not yt_cookies.get('SAPISID'):
 446                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 447                     self._set_cookie(
 448                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 449             else:
 450                 self._SAPISID = False
 451         if not self._SAPISID:
 452             return None
 453         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 454         sapisidhash = hashlib.sha1(
 455             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 456         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 457
 458     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 459                   note='Downloading API JSON', errnote='Unable to download API page',
 460                   context=None, api_key=None, api_hostname=None, default_client='web'):
 461
 462         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 463         data.update(query)
 464         real_headers = self.generate_api_headers(default_client=default_client)
 465         real_headers.update({'content-type': 'application/json'})
 466         if headers:
 467             real_headers.update(headers)
 468         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 469                    or api_key or self._extract_api_key(default_client=default_client))
 470         return self._download_json(
 471             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 472             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 473             data=json.dumps(data).encode('utf8'), headers=real_headers,
 474             query={'key': api_key, 'prettyPrint': 'false'})
 475
 476     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 477         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 478
 479     @staticmethod
 480     def _extract_session_index(*data):
 481         """
 482         Index of current account in account list.
 483         See: https://github.com/yt-dlp/yt-dlp/pull/519
 484         """
 485         for ytcfg in data:
 486             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 487             if session_index is not None:
 488                 return session_index
 489
 490     # Deprecated?
 491     def _extract_identity_token(self, ytcfg=None, webpage=None):
 492         if ytcfg:
 493             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 494             if token:
 495                 return token
 496         if webpage:
 497             return self._search_regex(
 498                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 499                 'identity token', default=None, fatal=False)
 500
 501     @staticmethod
 502     def _extract_account_syncid(*args):
 503         """
 504         Extract syncId required to download private playlists of secondary channels
 505         @params response and/or ytcfg
 506         """
 507         for data in args:
 508             # ytcfg includes channel_syncid if on secondary channel
 509             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 510             if delegated_sid:
 511                 return delegated_sid
 512             sync_ids = (try_get(
 513                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 514                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 515             if len(sync_ids) >= 2 and sync_ids[1]:
 516                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 517                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 518                 return sync_ids[0]
 519
 520     @staticmethod
 521     def _extract_visitor_data(*args):
 522         """
 523         Extracts visitorData from an API response or ytcfg
 524         Appears to be used to track session state
 525         """
 526         return get_first(
 527             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 528             expected_type=str)
 529
 530     @functools.cached_property
 531     def is_authenticated(self):
 532         return bool(self._generate_sapisidhash_header())
 533
 534     def extract_ytcfg(self, video_id, webpage):
 535         if not webpage:
 536             return {}
 537         return self._parse_json(
 538             self._search_regex(
 539                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 540                 default='{}'), video_id, fatal=False) or {}
 541
 542     def generate_api_headers(
 543             self, *, ytcfg=None, account_syncid=None, session_index=None,
 544             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 545
 546         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 547         headers = {
 548             'X-YouTube-Client-Name': str(
 549                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 550             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 551             'Origin': origin,
 552             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 553             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 554             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 555         }
 556         if session_index is None:
 557             session_index = self._extract_session_index(ytcfg)
 558         if account_syncid or session_index is not None:
 559             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 560
 561         auth = self._generate_sapisidhash_header(origin)
 562         if auth is not None:
 563             headers['Authorization'] = auth
 564             headers['X-Origin'] = origin
 565         return {h: v for h, v in headers.items() if v is not None}
 566
 567     def _download_ytcfg(self, client, video_id):
 568         url = {
 569             'web': 'https://www.youtube.com',
 570             'web_music': 'https://music.youtube.com',
 571             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 572         }.get(client)
 573         if not url:
 574             return {}
 575         webpage = self._download_webpage(
 576             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 577         return self.extract_ytcfg(video_id, webpage) or {}
 578
 579     @staticmethod
 580     def _build_api_continuation_query(continuation, ctp=None):
 581         query = {
 582             'continuation': continuation
 583         }
 584         # TODO: Inconsistency with clickTrackingParams.
 585         # Currently we have a fixed ctp contained within context (from ytcfg)
 586         # and a ctp in root query for continuation.
 587         if ctp:
 588             query['clickTracking'] = {'clickTrackingParams': ctp}
 589         return query
 590
 591     @classmethod
 592     def _extract_next_continuation_data(cls, renderer):
 593         next_continuation = try_get(
 594             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 595                        lambda x: x['continuation']['reloadContinuationData']), dict)
 596         if not next_continuation:
 597             return
 598         continuation = next_continuation.get('continuation')
 599         if not continuation:
 600             return
 601         ctp = next_continuation.get('clickTrackingParams')
 602         return cls._build_api_continuation_query(continuation, ctp)
 603
 604     @classmethod
 605     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 606         if isinstance(continuation_ep, dict):
 607             continuation = try_get(
 608                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 609             if not continuation:
 610                 return
 611             ctp = continuation_ep.get('clickTrackingParams')
 612             return cls._build_api_continuation_query(continuation, ctp)
 613
 614     @classmethod
 615     def _extract_continuation(cls, renderer):
 616         next_continuation = cls._extract_next_continuation_data(renderer)
 617         if next_continuation:
 618             return next_continuation
 619
 620         contents = []
 621         for key in ('contents', 'items'):
 622             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 623
 624         for content in contents:
 625             if not isinstance(content, dict):
 626                 continue
 627             continuation_ep = try_get(
 628                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 629                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 630                 dict)
 631             continuation = cls._extract_continuation_ep_data(continuation_ep)
 632             if continuation:
 633                 return continuation
 634
 635     @classmethod
 636     def _extract_alerts(cls, data):
 637         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 638             if not isinstance(alert_dict, dict):
 639                 continue
 640             for alert in alert_dict.values():
 641                 alert_type = alert.get('type')
 642                 if not alert_type:
 643                     continue
 644                 message = cls._get_text(alert, 'text')
 645                 if message:
 646                     yield alert_type, message
 647
 648     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 649         errors = []
 650         warnings = []
 651         for alert_type, alert_message in alerts:
 652             if alert_type.lower() == 'error' and fatal:
 653                 errors.append([alert_type, alert_message])
 654             else:
 655                 warnings.append([alert_type, alert_message])
 656
 657         for alert_type, alert_message in (warnings + errors[:-1]):
 658             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 659         if errors:
 660             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 661
 662     def _extract_and_report_alerts(self, data, *args, **kwargs):
 663         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 664
 665     def _extract_badges(self, renderer: dict):
 666         badges = set()
 667         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 668             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], str)
 669             if label:
 670                 badges.add(label.lower())
 671         return badges
 672
 673     @staticmethod
 674     def _get_text(data, *path_list, max_runs=None):
 675         for path in path_list or [None]:
 676             if path is None:
 677                 obj = [data]
 678             else:
 679                 obj = traverse_obj(data, path, default=[])
 680                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 681                     obj = [obj]
 682             for item in obj:
 683                 text = try_get(item, lambda x: x['simpleText'], str)
 684                 if text:
 685                     return text
 686                 runs = try_get(item, lambda x: x['runs'], list) or []
 687                 if not runs and isinstance(item, list):
 688                     runs = item
 689
 690                 runs = runs[:min(len(runs), max_runs or len(runs))]
 691                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 692                 if text:
 693                     return text
 694
 695     def _get_count(self, data, *path_list):
 696         count_text = self._get_text(data, *path_list) or ''
 697         count = parse_count(count_text)
 698         if count is None:
 699             count = str_to_int(
 700                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 701         return count
 702
 703     @staticmethod
 704     def _extract_thumbnails(data, *path_list):
 705         """
 706         Extract thumbnails from thumbnails dict
 707         @param path_list: path list to level that contains 'thumbnails' key
 708         """
 709         thumbnails = []
 710         for path in path_list or [()]:
 711             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 712                 thumbnail_url = url_or_none(thumbnail.get('url'))
 713                 if not thumbnail_url:
 714                     continue
 715                 # Sometimes youtube gives a wrong thumbnail URL. See:
 716                 # https://github.com/yt-dlp/yt-dlp/issues/233
 717                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 718                 if 'maxresdefault' in thumbnail_url:
 719                     thumbnail_url = thumbnail_url.split('?')[0]
 720                 thumbnails.append({
 721                     'url': thumbnail_url,
 722                     'height': int_or_none(thumbnail.get('height')),
 723                     'width': int_or_none(thumbnail.get('width')),
 724                 })
 725         return thumbnails
 726
 727     @staticmethod
 728     def extract_relative_time(relative_time_text):
 729         """
 730         Extracts a relative time from string and converts to dt object
 731         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 732         """
 733         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 734         if mobj:
 735             start = mobj.group('start')
 736             if start:
 737                 return datetime_from_str(start)
 738             try:
 739                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 740             except ValueError:
 741                 return None
 742
 743     def _extract_time_text(self, renderer, *path_list):
 744         """@returns (timestamp, time_text)"""
 745         text = self._get_text(renderer, *path_list) or ''
 746         dt = self.extract_relative_time(text)
 747         timestamp = None
 748         if isinstance(dt, datetime.datetime):
 749             timestamp = calendar.timegm(dt.timetuple())
 750
 751         if timestamp is None:
 752             timestamp = (
 753                 unified_timestamp(text) or unified_timestamp(
 754                     self._search_regex(
 755                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 756                         text.lower(), 'time text', default=None)))
 757
 758         if text and timestamp is None:
 759             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 760         return timestamp, text
 761
 762     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 763                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 764                           default_client='web'):
 765         response = None
 766         last_error = None
 767         count = -1
 768         retries = self.get_param('extractor_retries', 3)
 769         if check_get_keys is None:
 770             check_get_keys = []
 771         while count < retries:
 772             count += 1
 773             if last_error:
 774                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 775             try:
 776                 response = self._call_api(
 777                     ep=ep, fatal=True, headers=headers,
 778                     video_id=item_id, query=query,
 779                     context=self._extract_context(ytcfg, default_client),
 780                     api_key=self._extract_api_key(ytcfg, default_client),
 781                     api_hostname=api_hostname, default_client=default_client,
 782                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 783             except ExtractorError as e:
 784                 if isinstance(e.cause, network_exceptions):
 785                     if isinstance(e.cause, urllib.error.HTTPError):
 786                         first_bytes = e.cause.read(512)
 787                         if not is_html(first_bytes):
 788                             yt_error = try_get(
 789                                 self._parse_json(
 790                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 791                                 lambda x: x['error']['message'], str)
 792                             if yt_error:
 793                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 794                     # Downloading page may result in intermittent 5xx HTTP error
 795                     # Sometimes a 404 is also received. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 796                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 797                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 798                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
 799                         last_error = error_to_compat_str(e.cause or e.msg)
 800                         if count < retries:
 801                             continue
 802                 if fatal:
 803                     raise
 804                 else:
 805                     self.report_warning(error_to_compat_str(e))
 806                     return
 807
 808             else:
 809                 try:
 810                     self._extract_and_report_alerts(response, only_once=True)
 811                 except ExtractorError as e:
 812                     # YouTube servers may return errors we want to retry on in a 200 OK response
 813                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 814                     if 'unknown error' in e.msg.lower():
 815                         last_error = e.msg
 816                         continue
 817                     if fatal:
 818                         raise
 819                     self.report_warning(error_to_compat_str(e))
 820                     return
 821                 if not check_get_keys or dict_get(response, check_get_keys):
 822                     break
 823                 # Youtube sometimes sends incomplete data
 824                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 825                 last_error = 'Incomplete data received'
 826                 if count >= retries:
 827                     if fatal:
 828                         raise ExtractorError(last_error)
 829                     else:
 830                         self.report_warning(last_error)
 831                         return
 832         return response
 833
 834     @staticmethod
 835     def is_music_url(url):
 836         return re.match(r'https?://music\.youtube\.com/', url) is not None
 837
 838     def _extract_video(self, renderer):
 839         video_id = renderer.get('videoId')
 840         title = self._get_text(renderer, 'title')
 841         description = self._get_text(renderer, 'descriptionSnippet')
 842         duration = parse_duration(self._get_text(
 843             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 844         if duration is None:
 845             duration = parse_duration(self._search_regex(
 846                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 847                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 848                 video_id, default=None, group='duration'))
 849
 850         view_count = self._get_count(renderer, 'viewCountText')
 851
 852         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 853         channel_id = traverse_obj(
 854             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 855             expected_type=str, get_all=False)
 856         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 857         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 858         overlay_style = traverse_obj(
 859             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 860             get_all=False, expected_type=str)
 861         badges = self._extract_badges(renderer)
 862         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 863         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 864             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 865             expected_type=str)) or ''
 866         url = f'https://www.youtube.com/watch?v={video_id}'
 867         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 868             url = f'https://www.youtube.com/shorts/{video_id}'
 869
 870         return {
 871             '_type': 'url',
 872             'ie_key': YoutubeIE.ie_key(),
 873             'id': video_id,
 874             'url': url,
 875             'title': title,
 876             'description': description,
 877             'duration': duration,
 878             'view_count': view_count,
 879             'uploader': uploader,
 880             'channel_id': channel_id,
 881             'thumbnails': thumbnails,
 882             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 883                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 884                             else None),
 885             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 886                             else 'was_live' if 'streamed' in time_text.lower()
 887                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 888                             else None),
 889             'release_timestamp': scheduled_timestamp,
 890             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 891         }
 892
 893
 894 class YoutubeIE(YoutubeBaseInfoExtractor):
 895     IE_DESC = 'YouTube'
 896     _VALID_URL = r"""(?x)^
 897                      (
 898                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 899                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 900                             (?:www\.)?deturl\.com/www\.youtube\.com|
 901                             (?:www\.)?pwnyoutube\.com|
 902                             (?:www\.)?hooktube\.com|
 903                             (?:www\.)?yourepeat\.com|
 904                             tube\.majestyc\.net|
 905                             %(invidious)s|
 906                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 907                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 908                          (?:                                                  # the various things that can precede the ID:
 909                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 910                              |(?:                                             # or the v= param in all its forms
 911                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 912                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 913                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 914                                  v=
 915                              )
 916                          ))
 917                          |(?:
 918                             youtu\.be|                                        # just youtu.be/xxxx
 919                             vid\.plus|                                        # or vid.plus/xxxx
 920                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 921                             %(invidious)s
 922                          )/
 923                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 924                          )
 925                      )?                                                       # all until now is optional -> you can pass the naked ID
 926                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 927                      (?(1).+)?                                                # if we found the ID, everything can follow
 928                      (?:\#|$)""" % {
 929         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 930     }
 931     _PLAYER_INFO_RE = (
 932         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 933         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 934         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 935     )
 936     _formats = {
 937         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 938         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 939         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 940         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 941         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 942         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 943         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 944         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 945         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 946         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 947         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 948         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 949         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 950         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 951         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 952         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 953         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 954         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 955
 956
 957         # 3D videos
 958         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 959         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 960         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 961         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 962         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 963         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 964         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 965
 966         # Apple HTTP Live Streaming
 967         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 968         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 969         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 970         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 971         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 972         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 973         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 974         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 975
 976         # DASH mp4 video
 977         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 978         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 979         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 980         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 981         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 982         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 983         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 984         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 985         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 986         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 987         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 988         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 989
 990         # Dash mp4 audio
 991         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 992         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 993         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 994         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 995         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 996         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 997         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 998
 999         # Dash webm
1000         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1001         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1002         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1003         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1004         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1005         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1006         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1007         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1008         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1009         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1010         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1011         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1012         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1013         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1014         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1015         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1016         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1017         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1018         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1019         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1020         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1021         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1022
1023         # Dash webm audio
1024         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1025         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1026
1027         # Dash webm audio with opus inside
1028         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1029         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1030         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1031
1032         # RTMP (unnamed)
1033         '_rtmp': {'protocol': 'rtmp'},
1034
1035         # av01 video only formats sometimes served with "unknown" codecs
1036         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1037         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1038         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1039         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1040         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1041         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1042         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1043         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1044     }
1045     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1046
1047     _GEO_BYPASS = False
1048
1049     IE_NAME = 'youtube'
1050     _TESTS = [
1051         {
1052             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1053             'info_dict': {
1054                 'id': 'BaW_jenozKc',
1055                 'ext': 'mp4',
1056                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1057                 'uploader': 'Philipp Hagemeister',
1058                 'uploader_id': 'phihag',
1059                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1060                 'channel': 'Philipp Hagemeister',
1061                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1062                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1063                 'upload_date': '20121002',
1064                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1065                 'categories': ['Science & Technology'],
1066                 'tags': ['youtube-dl'],
1067                 'duration': 10,
1068                 'view_count': int,
1069                 'like_count': int,
1070                 'availability': 'public',
1071                 'playable_in_embed': True,
1072                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1073                 'live_status': 'not_live',
1074                 'age_limit': 0,
1075                 'start_time': 1,
1076                 'end_time': 9,
1077                 'comment_count': int,
1078                 'channel_follower_count': int
1079             }
1080         },
1081         {
1082             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1083             'note': 'Embed-only video (#1746)',
1084             'info_dict': {
1085                 'id': 'yZIXLfi8CZQ',
1086                 'ext': 'mp4',
1087                 'upload_date': '20120608',
1088                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1089                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1090                 'uploader': 'SET India',
1091                 'uploader_id': 'setindia',
1092                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1093                 'age_limit': 18,
1094             },
1095             'skip': 'Private video',
1096         },
1097         {
1098             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1099             'note': 'Use the first video ID in the URL',
1100             'info_dict': {
1101                 'id': 'BaW_jenozKc',
1102                 'ext': 'mp4',
1103                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1104                 'uploader': 'Philipp Hagemeister',
1105                 'uploader_id': 'phihag',
1106                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1107                 'channel': 'Philipp Hagemeister',
1108                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1109                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1110                 'upload_date': '20121002',
1111                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1112                 'categories': ['Science & Technology'],
1113                 'tags': ['youtube-dl'],
1114                 'duration': 10,
1115                 'view_count': int,
1116                 'like_count': int,
1117                 'availability': 'public',
1118                 'playable_in_embed': True,
1119                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1120                 'live_status': 'not_live',
1121                 'age_limit': 0,
1122                 'comment_count': int,
1123                 'channel_follower_count': int
1124             },
1125             'params': {
1126                 'skip_download': True,
1127             },
1128         },
1129         {
1130             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1131             'note': '256k DASH audio (format 141) via DASH manifest',
1132             'info_dict': {
1133                 'id': 'a9LDPn-MO4I',
1134                 'ext': 'm4a',
1135                 'upload_date': '20121002',
1136                 'uploader_id': '8KVIDEO',
1137                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1138                 'description': '',
1139                 'uploader': '8KVIDEO',
1140                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1141             },
1142             'params': {
1143                 'youtube_include_dash_manifest': True,
1144                 'format': '141',
1145             },
1146             'skip': 'format 141 not served anymore',
1147         },
1148         # DASH manifest with encrypted signature
1149         {
1150             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1151             'info_dict': {
1152                 'id': 'IB3lcPjvWLA',
1153                 'ext': 'm4a',
1154                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1155                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1156                 'duration': 244,
1157                 'uploader': 'AfrojackVEVO',
1158                 'uploader_id': 'AfrojackVEVO',
1159                 'upload_date': '20131011',
1160                 'abr': 129.495,
1161                 'like_count': int,
1162                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1163                 'playable_in_embed': True,
1164                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1165                 'view_count': int,
1166                 'track': 'The Spark',
1167                 'live_status': 'not_live',
1168                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1169                 'channel': 'Afrojack',
1170                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1171                 'tags': 'count:19',
1172                 'availability': 'public',
1173                 'categories': ['Music'],
1174                 'age_limit': 0,
1175                 'alt_title': 'The Spark',
1176                 'channel_follower_count': int
1177             },
1178             'params': {
1179                 'youtube_include_dash_manifest': True,
1180                 'format': '141/bestaudio[ext=m4a]',
1181             },
1182         },
1183         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1184         {
1185             'note': 'Embed allowed age-gate video',
1186             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1187             'info_dict': {
1188                 'id': 'HtVdAasjOgU',
1189                 'ext': 'mp4',
1190                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1191                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1192                 'duration': 142,
1193                 'uploader': 'The Witcher',
1194                 'uploader_id': 'WitcherGame',
1195                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1196                 'upload_date': '20140605',
1197                 'age_limit': 18,
1198                 'categories': ['Gaming'],
1199                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1200                 'availability': 'needs_auth',
1201                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1202                 'like_count': int,
1203                 'channel': 'The Witcher',
1204                 'live_status': 'not_live',
1205                 'tags': 'count:17',
1206                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1207                 'playable_in_embed': True,
1208                 'view_count': int,
1209                 'channel_follower_count': int
1210             },
1211         },
1212         {
1213             'note': 'Age-gate video with embed allowed in public site',
1214             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1215             'info_dict': {
1216                 'id': 'HsUATh_Nc2U',
1217                 'ext': 'mp4',
1218                 'title': 'Godzilla 2 (Official Video)',
1219                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1220                 'upload_date': '20200408',
1221                 'uploader_id': 'FlyingKitty900',
1222                 'uploader': 'FlyingKitty',
1223                 'age_limit': 18,
1224                 'availability': 'needs_auth',
1225                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1226                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1227                 'channel': 'FlyingKitty',
1228                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1229                 'view_count': int,
1230                 'categories': ['Entertainment'],
1231                 'live_status': 'not_live',
1232                 'tags': ['Flyingkitty', 'godzilla 2'],
1233                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1234                 'like_count': int,
1235                 'duration': 177,
1236                 'playable_in_embed': True,
1237                 'channel_follower_count': int
1238             },
1239         },
1240         {
1241             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1242             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1243             'info_dict': {
1244                 'id': 'Tq92D6wQ1mg',
1245                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1246                 'ext': 'mp4',
1247                 'upload_date': '20191228',
1248                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1249                 'uploader': 'Projekt Melody',
1250                 'description': 'md5:17eccca93a786d51bc67646756894066',
1251                 'age_limit': 18,
1252                 'like_count': int,
1253                 'availability': 'needs_auth',
1254                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1255                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1256                 'view_count': int,
1257                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1258                 'channel': 'Projekt Melody',
1259                 'live_status': 'not_live',
1260                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1261                 'playable_in_embed': True,
1262                 'categories': ['Entertainment'],
1263                 'duration': 106,
1264                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1265                 'comment_count': int,
1266                 'channel_follower_count': int
1267             },
1268         },
1269         {
1270             'note': 'Non-Agegated non-embeddable video',
1271             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1272             'info_dict': {
1273                 'id': 'MeJVWBSsPAY',
1274                 'ext': 'mp4',
1275                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1276                 'uploader': 'Herr Lurik',
1277                 'uploader_id': 'st3in234',
1278                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1279                 'upload_date': '20130730',
1280                 'track': 'Such mich find mich',
1281                 'age_limit': 0,
1282                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1283                 'like_count': int,
1284                 'playable_in_embed': False,
1285                 'creator': 'OOMPH!',
1286                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1287                 'view_count': int,
1288                 'alt_title': 'Such mich find mich',
1289                 'duration': 210,
1290                 'channel': 'Herr Lurik',
1291                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1292                 'categories': ['Music'],
1293                 'availability': 'public',
1294                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1295                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1296                 'live_status': 'not_live',
1297                 'artist': 'OOMPH!',
1298                 'channel_follower_count': int
1299             },
1300         },
1301         {
1302             'note': 'Non-bypassable age-gated video',
1303             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1304             'only_matching': True,
1305         },
1306         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1307         # YouTube Red ad is not captured for creator
1308         {
1309             'url': '__2ABJjxzNo',
1310             'info_dict': {
1311                 'id': '__2ABJjxzNo',
1312                 'ext': 'mp4',
1313                 'duration': 266,
1314                 'upload_date': '20100430',
1315                 'uploader_id': 'deadmau5',
1316                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1317                 'creator': 'deadmau5',
1318                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1319                 'uploader': 'deadmau5',
1320                 'title': 'Deadmau5 - Some Chords (HD)',
1321                 'alt_title': 'Some Chords',
1322                 'availability': 'public',
1323                 'tags': 'count:14',
1324                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1325                 'view_count': int,
1326                 'live_status': 'not_live',
1327                 'channel': 'deadmau5',
1328                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1329                 'like_count': int,
1330                 'track': 'Some Chords',
1331                 'artist': 'deadmau5',
1332                 'playable_in_embed': True,
1333                 'age_limit': 0,
1334                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1335                 'categories': ['Music'],
1336                 'album': 'Some Chords',
1337                 'channel_follower_count': int
1338             },
1339             'expected_warnings': [
1340                 'DASH manifest missing',
1341             ]
1342         },
1343         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1344         {
1345             'url': 'lqQg6PlCWgI',
1346             'info_dict': {
1347                 'id': 'lqQg6PlCWgI',
1348                 'ext': 'mp4',
1349                 'duration': 6085,
1350                 'upload_date': '20150827',
1351                 'uploader_id': 'olympic',
1352                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1353                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1354                 'uploader': 'Olympics',
1355                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1356                 'like_count': int,
1357                 'release_timestamp': 1343767800,
1358                 'playable_in_embed': True,
1359                 'categories': ['Sports'],
1360                 'release_date': '20120731',
1361                 'channel': 'Olympics',
1362                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1363                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1364                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1365                 'age_limit': 0,
1366                 'availability': 'public',
1367                 'live_status': 'was_live',
1368                 'view_count': int,
1369                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1370                 'channel_follower_count': int
1371             },
1372             'params': {
1373                 'skip_download': 'requires avconv',
1374             }
1375         },
1376         # Non-square pixels
1377         {
1378             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1379             'info_dict': {
1380                 'id': '_b-2C3KPAM0',
1381                 'ext': 'mp4',
1382                 'stretched_ratio': 16 / 9.,
1383                 'duration': 85,
1384                 'upload_date': '20110310',
1385                 'uploader_id': 'AllenMeow',
1386                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1387                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1388                 'uploader': '孫ᄋᄅ',
1389                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1390                 'playable_in_embed': True,
1391                 'channel': '孫ᄋᄅ',
1392                 'age_limit': 0,
1393                 'tags': 'count:11',
1394                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1395                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1396                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1397                 'view_count': int,
1398                 'categories': ['People & Blogs'],
1399                 'like_count': int,
1400                 'live_status': 'not_live',
1401                 'availability': 'unlisted',
1402                 'comment_count': int,
1403                 'channel_follower_count': int
1404             },
1405         },
1406         # url_encoded_fmt_stream_map is empty string
1407         {
1408             'url': 'qEJwOuvDf7I',
1409             'info_dict': {
1410                 'id': 'qEJwOuvDf7I',
1411                 'ext': 'webm',
1412                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1413                 'description': '',
1414                 'upload_date': '20150404',
1415                 'uploader_id': 'spbelect',
1416                 'uploader': 'Наблюдатели Петербурга',
1417             },
1418             'params': {
1419                 'skip_download': 'requires avconv',
1420             },
1421             'skip': 'This live event has ended.',
1422         },
1423         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1424         {
1425             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1426             'info_dict': {
1427                 'id': 'FIl7x6_3R5Y',
1428                 'ext': 'webm',
1429                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1430                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1431                 'duration': 220,
1432                 'upload_date': '20150625',
1433                 'uploader_id': 'dorappi2000',
1434                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1435                 'uploader': 'dorappi2000',
1436                 'formats': 'mincount:31',
1437             },
1438             'skip': 'not actual anymore',
1439         },
1440         # DASH manifest with segment_list
1441         {
1442             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1443             'md5': '8ce563a1d667b599d21064e982ab9e31',
1444             'info_dict': {
1445                 'id': 'CsmdDsKjzN8',
1446                 'ext': 'mp4',
1447                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1448                 'uploader': 'Airtek',
1449                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1450                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1451                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1452             },
1453             'params': {
1454                 'youtube_include_dash_manifest': True,
1455                 'format': '135',  # bestvideo
1456             },
1457             'skip': 'This live event has ended.',
1458         },
1459         {
1460             # Multifeed videos (multiple cameras), URL is for Main Camera
1461             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1462             'info_dict': {
1463                 'id': 'jvGDaLqkpTg',
1464                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1465                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1466             },
1467             'playlist': [{
1468                 'info_dict': {
1469                     'id': 'jvGDaLqkpTg',
1470                     'ext': 'mp4',
1471                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1472                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1473                     'duration': 10643,
1474                     'upload_date': '20161111',
1475                     'uploader': 'Team PGP',
1476                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1477                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1478                 },
1479             }, {
1480                 'info_dict': {
1481                     'id': '3AKt1R1aDnw',
1482                     'ext': 'mp4',
1483                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1484                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1485                     'duration': 10991,
1486                     'upload_date': '20161111',
1487                     'uploader': 'Team PGP',
1488                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1489                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1490                 },
1491             }, {
1492                 'info_dict': {
1493                     'id': 'RtAMM00gpVc',
1494                     'ext': 'mp4',
1495                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1496                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1497                     'duration': 10995,
1498                     'upload_date': '20161111',
1499                     'uploader': 'Team PGP',
1500                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1501                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1502                 },
1503             }, {
1504                 'info_dict': {
1505                     'id': '6N2fdlP3C5U',
1506                     'ext': 'mp4',
1507                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1508                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1509                     'duration': 10990,
1510                     'upload_date': '20161111',
1511                     'uploader': 'Team PGP',
1512                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1513                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1514                 },
1515             }],
1516             'params': {
1517                 'skip_download': True,
1518             },
1519             'skip': 'Not multifeed anymore',
1520         },
1521         {
1522             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1523             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1524             'info_dict': {
1525                 'id': 'gVfLd0zydlo',
1526                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1527             },
1528             'playlist_count': 2,
1529             'skip': 'Not multifeed anymore',
1530         },
1531         {
1532             'url': 'https://vid.plus/FlRa-iH7PGw',
1533             'only_matching': True,
1534         },
1535         {
1536             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1537             'only_matching': True,
1538         },
1539         {
1540             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1541             # Also tests cut-off URL expansion in video description (see
1542             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1543             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1544             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1545             'info_dict': {
1546                 'id': 'lsguqyKfVQg',
1547                 'ext': 'mp4',
1548                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1549                 'alt_title': 'Dark Walk',
1550                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1551                 'duration': 133,
1552                 'upload_date': '20151119',
1553                 'uploader_id': 'IronSoulElf',
1554                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1555                 'uploader': 'IronSoulElf',
1556                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1557                 'track': 'Dark Walk',
1558                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1559                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1560                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1561                 'categories': ['Film & Animation'],
1562                 'view_count': int,
1563                 'live_status': 'not_live',
1564                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1565                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1566                 'tags': 'count:13',
1567                 'availability': 'public',
1568                 'channel': 'IronSoulElf',
1569                 'playable_in_embed': True,
1570                 'like_count': int,
1571                 'age_limit': 0,
1572                 'channel_follower_count': int
1573             },
1574             'params': {
1575                 'skip_download': True,
1576             },
1577         },
1578         {
1579             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1580             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1581             'only_matching': True,
1582         },
1583         {
1584             # Video with yt:stretch=17:0
1585             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1586             'info_dict': {
1587                 'id': 'Q39EVAstoRM',
1588                 'ext': 'mp4',
1589                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1590                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1591                 'upload_date': '20151107',
1592                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1593                 'uploader': 'CH GAMER DROID',
1594             },
1595             'params': {
1596                 'skip_download': True,
1597             },
1598             'skip': 'This video does not exist.',
1599         },
1600         {
1601             # Video with incomplete 'yt:stretch=16:'
1602             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1603             'only_matching': True,
1604         },
1605         {
1606             # Video licensed under Creative Commons
1607             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1608             'info_dict': {
1609                 'id': 'M4gD1WSo5mA',
1610                 'ext': 'mp4',
1611                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1612                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1613                 'duration': 721,
1614                 'upload_date': '20150128',
1615                 'uploader_id': 'BerkmanCenter',
1616                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1617                 'uploader': 'The Berkman Klein Center for Internet & Society',
1618                 'license': 'Creative Commons Attribution license (reuse allowed)',
1619                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1620                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1621                 'like_count': int,
1622                 'age_limit': 0,
1623                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1624                 'channel': 'The Berkman Klein Center for Internet & Society',
1625                 'availability': 'public',
1626                 'view_count': int,
1627                 'categories': ['Education'],
1628                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1629                 'live_status': 'not_live',
1630                 'playable_in_embed': True,
1631                 'comment_count': int,
1632                 'channel_follower_count': int
1633             },
1634             'params': {
1635                 'skip_download': True,
1636             },
1637         },
1638         {
1639             # Channel-like uploader_url
1640             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1641             'info_dict': {
1642                 'id': 'eQcmzGIKrzg',
1643                 'ext': 'mp4',
1644                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1645                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1646                 'duration': 4060,
1647                 'upload_date': '20151120',
1648                 'uploader': 'Bernie Sanders',
1649                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1650                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1651                 'license': 'Creative Commons Attribution license (reuse allowed)',
1652                 'playable_in_embed': True,
1653                 'tags': 'count:12',
1654                 'like_count': int,
1655                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1656                 'age_limit': 0,
1657                 'availability': 'public',
1658                 'categories': ['News & Politics'],
1659                 'channel': 'Bernie Sanders',
1660                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1661                 'view_count': int,
1662                 'live_status': 'not_live',
1663                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1664                 'comment_count': int,
1665                 'channel_follower_count': int
1666             },
1667             'params': {
1668                 'skip_download': True,
1669             },
1670         },
1671         {
1672             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1673             'only_matching': True,
1674         },
1675         {
1676             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1677             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1678             'only_matching': True,
1679         },
1680         {
1681             # Rental video preview
1682             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1683             'info_dict': {
1684                 'id': 'uGpuVWrhIzE',
1685                 'ext': 'mp4',
1686                 'title': 'Piku - Trailer',
1687                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1688                 'upload_date': '20150811',
1689                 'uploader': 'FlixMatrix',
1690                 'uploader_id': 'FlixMatrixKaravan',
1691                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1692                 'license': 'Standard YouTube License',
1693             },
1694             'params': {
1695                 'skip_download': True,
1696             },
1697             'skip': 'This video is not available.',
1698         },
1699         {
1700             # YouTube Red video with episode data
1701             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1702             'info_dict': {
1703                 'id': 'iqKdEhx-dD4',
1704                 'ext': 'mp4',
1705                 'title': 'Isolation - Mind Field (Ep 1)',
1706                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1707                 'duration': 2085,
1708                 'upload_date': '20170118',
1709                 'uploader': 'Vsauce',
1710                 'uploader_id': 'Vsauce',
1711                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1712                 'series': 'Mind Field',
1713                 'season_number': 1,
1714                 'episode_number': 1,
1715                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1716                 'tags': 'count:12',
1717                 'view_count': int,
1718                 'availability': 'public',
1719                 'age_limit': 0,
1720                 'channel': 'Vsauce',
1721                 'episode': 'Episode 1',
1722                 'categories': ['Entertainment'],
1723                 'season': 'Season 1',
1724                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1725                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1726                 'like_count': int,
1727                 'playable_in_embed': True,
1728                 'live_status': 'not_live',
1729                 'channel_follower_count': int
1730             },
1731             'params': {
1732                 'skip_download': True,
1733             },
1734             'expected_warnings': [
1735                 'Skipping DASH manifest',
1736             ],
1737         },
1738         {
1739             # The following content has been identified by the YouTube community
1740             # as inappropriate or offensive to some audiences.
1741             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1742             'info_dict': {
1743                 'id': '6SJNVb0GnPI',
1744                 'ext': 'mp4',
1745                 'title': 'Race Differences in Intelligence',
1746                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1747                 'duration': 965,
1748                 'upload_date': '20140124',
1749                 'uploader': 'New Century Foundation',
1750                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1751                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1752             },
1753             'params': {
1754                 'skip_download': True,
1755             },
1756             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1757         },
1758         {
1759             # itag 212
1760             'url': '1t24XAntNCY',
1761             'only_matching': True,
1762         },
1763         {
1764             # geo restricted to JP
1765             'url': 'sJL6WA-aGkQ',
1766             'only_matching': True,
1767         },
1768         {
1769             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1770             'only_matching': True,
1771         },
1772         {
1773             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1774             'only_matching': True,
1775         },
1776         {
1777             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1778             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1779             'only_matching': True,
1780         },
1781         {
1782             # DRM protected
1783             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1784             'only_matching': True,
1785         },
1786         {
1787             # Video with unsupported adaptive stream type formats
1788             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1789             'info_dict': {
1790                 'id': 'Z4Vy8R84T1U',
1791                 'ext': 'mp4',
1792                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1793                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1794                 'duration': 433,
1795                 'upload_date': '20130923',
1796                 'uploader': 'Amelia Putri Harwita',
1797                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1798                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1799                 'formats': 'maxcount:10',
1800             },
1801             'params': {
1802                 'skip_download': True,
1803                 'youtube_include_dash_manifest': False,
1804             },
1805             'skip': 'not actual anymore',
1806         },
1807         {
1808             # Youtube Music Auto-generated description
1809             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1810             'info_dict': {
1811                 'id': 'MgNrAu2pzNs',
1812                 'ext': 'mp4',
1813                 'title': 'Voyeur Girl',
1814                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1815                 'upload_date': '20190312',
1816                 'uploader': 'Stephen - Topic',
1817                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1818                 'artist': 'Stephen',
1819                 'track': 'Voyeur Girl',
1820                 'album': 'it\'s too much love to know my dear',
1821                 'release_date': '20190313',
1822                 'release_year': 2019,
1823                 'alt_title': 'Voyeur Girl',
1824                 'view_count': int,
1825                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1826                 'playable_in_embed': True,
1827                 'like_count': int,
1828                 'categories': ['Music'],
1829                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1830                 'channel': 'Stephen',
1831                 'availability': 'public',
1832                 'creator': 'Stephen',
1833                 'duration': 169,
1834                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1835                 'age_limit': 0,
1836                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1837                 'tags': 'count:11',
1838                 'live_status': 'not_live',
1839                 'channel_follower_count': int
1840             },
1841             'params': {
1842                 'skip_download': True,
1843             },
1844         },
1845         {
1846             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1847             'only_matching': True,
1848         },
1849         {
1850             # invalid -> valid video id redirection
1851             'url': 'DJztXj2GPfl',
1852             'info_dict': {
1853                 'id': 'DJztXj2GPfk',
1854                 'ext': 'mp4',
1855                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1856                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1857                 'upload_date': '20090125',
1858                 'uploader': 'Prochorowka',
1859                 'uploader_id': 'Prochorowka',
1860                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1861                 'artist': 'Panjabi MC',
1862                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1863                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1864             },
1865             'params': {
1866                 'skip_download': True,
1867             },
1868             'skip': 'Video unavailable',
1869         },
1870         {
1871             # empty description results in an empty string
1872             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1873             'info_dict': {
1874                 'id': 'x41yOUIvK2k',
1875                 'ext': 'mp4',
1876                 'title': 'IMG 3456',
1877                 'description': '',
1878                 'upload_date': '20170613',
1879                 'uploader_id': 'ElevageOrVert',
1880                 'uploader': 'ElevageOrVert',
1881                 'view_count': int,
1882                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1883                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1884                 'like_count': int,
1885                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1886                 'tags': [],
1887                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1888                 'availability': 'public',
1889                 'age_limit': 0,
1890                 'categories': ['Pets & Animals'],
1891                 'duration': 7,
1892                 'playable_in_embed': True,
1893                 'live_status': 'not_live',
1894                 'channel': 'ElevageOrVert',
1895                 'channel_follower_count': int
1896             },
1897             'params': {
1898                 'skip_download': True,
1899             },
1900         },
1901         {
1902             # with '};' inside yt initial data (see [1])
1903             # see [2] for an example with '};' inside ytInitialPlayerResponse
1904             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1905             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1906             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1907             'info_dict': {
1908                 'id': 'CHqg6qOn4no',
1909                 'ext': 'mp4',
1910                 'title': 'Part 77   Sort a list of simple types in c#',
1911                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1912                 'upload_date': '20130831',
1913                 'uploader_id': 'kudvenkat',
1914                 'uploader': 'kudvenkat',
1915                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1916                 'like_count': int,
1917                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1918                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1919                 'live_status': 'not_live',
1920                 'categories': ['Education'],
1921                 'availability': 'public',
1922                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1923                 'tags': 'count:12',
1924                 'playable_in_embed': True,
1925                 'age_limit': 0,
1926                 'view_count': int,
1927                 'duration': 522,
1928                 'channel': 'kudvenkat',
1929                 'comment_count': int,
1930                 'channel_follower_count': int
1931             },
1932             'params': {
1933                 'skip_download': True,
1934             },
1935         },
1936         {
1937             # another example of '};' in ytInitialData
1938             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1939             'only_matching': True,
1940         },
1941         {
1942             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1943             'only_matching': True,
1944         },
1945         {
1946             # https://github.com/ytdl-org/youtube-dl/pull/28094
1947             'url': 'OtqTfy26tG0',
1948             'info_dict': {
1949                 'id': 'OtqTfy26tG0',
1950                 'ext': 'mp4',
1951                 'title': 'Burn Out',
1952                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1953                 'upload_date': '20141120',
1954                 'uploader': 'The Cinematic Orchestra - Topic',
1955                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1956                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1957                 'artist': 'The Cinematic Orchestra',
1958                 'track': 'Burn Out',
1959                 'album': 'Every Day',
1960                 'like_count': int,
1961                 'live_status': 'not_live',
1962                 'alt_title': 'Burn Out',
1963                 'duration': 614,
1964                 'age_limit': 0,
1965                 'view_count': int,
1966                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1967                 'creator': 'The Cinematic Orchestra',
1968                 'channel': 'The Cinematic Orchestra',
1969                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1970                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1971                 'availability': 'public',
1972                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1973                 'categories': ['Music'],
1974                 'playable_in_embed': True,
1975                 'channel_follower_count': int
1976             },
1977             'params': {
1978                 'skip_download': True,
1979             },
1980         },
1981         {
1982             # controversial video, only works with bpctr when authenticated with cookies
1983             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1984             'only_matching': True,
1985         },
1986         {
1987             # controversial video, requires bpctr/contentCheckOk
1988             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1989             'info_dict': {
1990                 'id': 'SZJvDhaSDnc',
1991                 'ext': 'mp4',
1992                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1993                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1994                 'uploader': 'CBS Mornings',
1995                 'uploader_id': 'CBSThisMorning',
1996                 'upload_date': '20140716',
1997                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1998                 'duration': 170,
1999                 'categories': ['News & Politics'],
2000                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2001                 'view_count': int,
2002                 'channel': 'CBS Mornings',
2003                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2004                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2005                 'age_limit': 18,
2006                 'availability': 'needs_auth',
2007                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2008                 'like_count': int,
2009                 'live_status': 'not_live',
2010                 'playable_in_embed': True,
2011                 'channel_follower_count': int
2012             }
2013         },
2014         {
2015             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2016             'url': 'cBvYw8_A0vQ',
2017             'info_dict': {
2018                 'id': 'cBvYw8_A0vQ',
2019                 'ext': 'mp4',
2020                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2021                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2022                 'upload_date': '20201120',
2023                 'uploader': 'Walk around Japan',
2024                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2025                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2026                 'duration': 1456,
2027                 'categories': ['Travel & Events'],
2028                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2029                 'view_count': int,
2030                 'channel': 'Walk around Japan',
2031                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2032                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2033                 'age_limit': 0,
2034                 'availability': 'public',
2035                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2036                 'live_status': 'not_live',
2037                 'playable_in_embed': True,
2038                 'channel_follower_count': int
2039             },
2040             'params': {
2041                 'skip_download': True,
2042             },
2043         }, {
2044             # Has multiple audio streams
2045             'url': 'WaOKSUlf4TM',
2046             'only_matching': True
2047         }, {
2048             # Requires Premium: has format 141 when requested using YTM url
2049             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2050             'only_matching': True
2051         }, {
2052             # multiple subtitles with same lang_code
2053             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2054             'only_matching': True,
2055         }, {
2056             # Force use android client fallback
2057             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2058             'info_dict': {
2059                 'id': 'YOelRv7fMxY',
2060                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2061                 'ext': '3gp',
2062                 'upload_date': '20210624',
2063                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2064                 'uploader': 'colinfurze',
2065                 'uploader_id': 'colinfurze',
2066                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2067                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2068                 'duration': 596,
2069                 'categories': ['Entertainment'],
2070                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2071                 'view_count': int,
2072                 'channel': 'colinfurze',
2073                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2074                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2075                 'age_limit': 0,
2076                 'availability': 'public',
2077                 'like_count': int,
2078                 'live_status': 'not_live',
2079                 'playable_in_embed': True,
2080                 'channel_follower_count': int
2081             },
2082             'params': {
2083                 'format': '17',  # 3gp format available on android
2084                 'extractor_args': {'youtube': {'player_client': ['android']}},
2085             },
2086         },
2087         {
2088             # Skip download of additional client configs (remix client config in this case)
2089             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2090             'only_matching': True,
2091             'params': {
2092                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2093             },
2094         }, {
2095             # shorts
2096             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2097             'only_matching': True,
2098         }, {
2099             'note': 'Storyboards',
2100             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2101             'info_dict': {
2102                 'id': '5KLPxDtMqe8',
2103                 'ext': 'mhtml',
2104                 'format_id': 'sb0',
2105                 'title': 'Your Brain is Plastic',
2106                 'uploader_id': 'scishow',
2107                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2108                 'upload_date': '20140324',
2109                 'uploader': 'SciShow',
2110                 'like_count': int,
2111                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2112                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2113                 'view_count': int,
2114                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2115                 'playable_in_embed': True,
2116                 'tags': 'count:12',
2117                 'uploader_url': 'http://www.youtube.com/user/scishow',
2118                 'availability': 'public',
2119                 'channel': 'SciShow',
2120                 'live_status': 'not_live',
2121                 'duration': 248,
2122                 'categories': ['Education'],
2123                 'age_limit': 0,
2124                 'channel_follower_count': int
2125             }, 'params': {'format': 'mhtml', 'skip_download': True}
2126         }, {
2127             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2128             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2129             'info_dict': {
2130                 'id': '2NUZ8W2llS4',
2131                 'ext': 'mp4',
2132                 'title': 'The NP that test your phone performance 🙂',
2133                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2134                 'uploader': 'Leon Nguyen',
2135                 'uploader_id': 'VNSXIII',
2136                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2137                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2138                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2139                 'duration': 21,
2140                 'view_count': int,
2141                 'age_limit': 0,
2142                 'categories': ['Gaming'],
2143                 'tags': 'count:23',
2144                 'playable_in_embed': True,
2145                 'live_status': 'not_live',
2146                 'upload_date': '20220103',
2147                 'like_count': int,
2148                 'availability': 'public',
2149                 'channel': 'Leon Nguyen',
2150                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2151                 'comment_count': int,
2152                 'channel_follower_count': int
2153             }
2154         }, {
2155             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2156             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2157             'info_dict': {
2158                 'id': 'mzZzzBU6lrM',
2159                 'ext': 'mp4',
2160                 'title': 'I Met GeorgeNotFound In Real Life...',
2161                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2162                 'uploader': 'Quackity',
2163                 'uploader_id': 'QuackityHQ',
2164                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2165                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2166                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2167                 'duration': 955,
2168                 'view_count': int,
2169                 'age_limit': 0,
2170                 'categories': ['Entertainment'],
2171                 'tags': 'count:26',
2172                 'playable_in_embed': True,
2173                 'live_status': 'not_live',
2174                 'release_timestamp': 1641172509,
2175                 'release_date': '20220103',
2176                 'upload_date': '20220103',
2177                 'like_count': int,
2178                 'availability': 'public',
2179                 'channel': 'Quackity',
2180                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2181                 'channel_follower_count': int
2182             }
2183         },
2184         {   # continuous livestream. Microformat upload date should be preferred.
2185             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2186             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2187             'info_dict': {
2188                 'id': 'kgx4WGK0oNU',
2189                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2190                 'ext': 'mp4',
2191                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2192                 'availability': 'public',
2193                 'age_limit': 0,
2194                 'release_timestamp': 1637975704,
2195                 'upload_date': '20210619',
2196                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2197                 'live_status': 'is_live',
2198                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2199                 'uploader': '阿鲍Abao',
2200                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2201                 'channel': 'Abao in Tokyo',
2202                 'channel_follower_count': int,
2203                 'release_date': '20211127',
2204                 'tags': 'count:39',
2205                 'categories': ['People & Blogs'],
2206                 'like_count': int,
2207                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2208                 'view_count': int,
2209                 'playable_in_embed': True,
2210                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2211             },
2212             'params': {'skip_download': True}
2213         }, {
2214             # Story. Requires specific player params to work.
2215             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2216             'info_dict': {
2217                 'id': 'vv8qTUWmulI',
2218                 'ext': 'mp4',
2219                 'availability': 'unlisted',
2220                 'view_count': int,
2221                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2222                 'upload_date': '20220526',
2223                 'categories': ['Education'],
2224                 'title': 'Story',
2225                 'channel': 'IT\'S HISTORY',
2226                 'description': '',
2227                 'uploader_id': 'BlastfromthePast',
2228                 'duration': 12,
2229                 'uploader': 'IT\'S HISTORY',
2230                 'playable_in_embed': True,
2231                 'age_limit': 0,
2232                 'live_status': 'not_live',
2233                 'tags': [],
2234                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2235                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2236                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2237             },
2238             'skip': 'stories get removed after some period of time',
2239         }, {
2240             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2241             'info_dict': {
2242                 'id': 'tjjjtzRLHvA',
2243                 'ext': 'mp4',
2244                 'title': 'ハッシュタグ無し };if window.ytcsi',
2245                 'upload_date': '20220323',
2246                 'like_count': int,
2247                 'availability': 'unlisted',
2248                 'channel': 'nao20010128nao',
2249                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2250                 'age_limit': 0,
2251                 'uploader': 'nao20010128nao',
2252                 'uploader_id': 'nao20010128nao',
2253                 'categories': ['Music'],
2254                 'view_count': int,
2255                 'description': '',
2256                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2257                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2258                 'live_status': 'not_live',
2259                 'playable_in_embed': True,
2260                 'channel_follower_count': int,
2261                 'duration': 6,
2262                 'tags': [],
2263                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2264             }
2265         }
2266     ]
2267
2268     @classmethod
2269     def suitable(cls, url):
2270         from ..utils import parse_qs
2271
2272         qs = parse_qs(url)
2273         if qs.get('list', [None])[0]:
2274             return False
2275         return super().suitable(url)
2276
2277     def __init__(self, *args, **kwargs):
2278         super().__init__(*args, **kwargs)
2279         self._code_cache = {}
2280         self._player_cache = {}
2281
2282     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2283         lock = threading.Lock()
2284
2285         is_live = True
2286         start_time = time.time()
2287         formats = [f for f in formats if f.get('is_from_start')]
2288
2289         def refetch_manifest(format_id, delay):
2290             nonlocal formats, start_time, is_live
2291             if time.time() <= start_time + delay:
2292                 return
2293
2294             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2295             video_details = traverse_obj(
2296                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2297             microformats = traverse_obj(
2298                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2299                 expected_type=dict, default=[])
2300             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2301             start_time = time.time()
2302
2303         def mpd_feed(format_id, delay):
2304             """
2305             @returns (manifest_url, manifest_stream_number, is_live) or None
2306             """
2307             with lock:
2308                 refetch_manifest(format_id, delay)
2309
2310             f = next((f for f in formats if f['format_id'] == format_id), None)
2311             if not f:
2312                 if not is_live:
2313                     self.to_screen(f'{video_id}: Video is no longer live')
2314                 else:
2315                     self.report_warning(
2316                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2317                 return None
2318             return f['manifest_url'], f['manifest_stream_number'], is_live
2319
2320         for f in formats:
2321             f['is_live'] = True
2322             f['protocol'] = 'http_dash_segments_generator'
2323             f['fragments'] = functools.partial(
2324                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2325
2326     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2327         FETCH_SPAN, MAX_DURATION = 5, 432000
2328
2329         mpd_url, stream_number, is_live = None, None, True
2330
2331         begin_index = 0
2332         download_start_time = ctx.get('start') or time.time()
2333
2334         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2335         if lack_early_segments:
2336             self.report_warning(bug_reports_message(
2337                 'Starting download from the last 120 hours of the live stream since '
2338                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2339             lack_early_segments = True
2340
2341         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2342         fragments, fragment_base_url = None, None
2343
2344         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2345             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2346             # Obtain from MPD's maximum seq value
2347             old_mpd_url = mpd_url
2348             last_error = ctx.pop('last_error', None)
2349             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2350             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2351                                                or (mpd_url, stream_number, False))
2352             if not refresh_sequence:
2353                 if expire_fast and not is_live:
2354                     return False, last_seq
2355                 elif old_mpd_url == mpd_url:
2356                     return True, last_seq
2357             try:
2358                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2359                     mpd_url, None, note=False, errnote=False, fatal=False)
2360             except ExtractorError:
2361                 fmts = None
2362             if not fmts:
2363                 no_fragment_score += 2
2364                 return False, last_seq
2365             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2366             fragments = fmt_info['fragments']
2367             fragment_base_url = fmt_info['fragment_base_url']
2368             assert fragment_base_url
2369
2370             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2371             return True, _last_seq
2372
2373         while is_live:
2374             fetch_time = time.time()
2375             if no_fragment_score > 30:
2376                 return
2377             if last_segment_url:
2378                 # Obtain from "X-Head-Seqnum" header value from each segment
2379                 try:
2380                     urlh = self._request_webpage(
2381                         last_segment_url, None, note=False, errnote=False, fatal=False)
2382                 except ExtractorError:
2383                     urlh = None
2384                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2385                 if last_seq is None:
2386                     no_fragment_score += 2
2387                     last_segment_url = None
2388                     continue
2389             else:
2390                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2391                 no_fragment_score += 2
2392                 if not should_continue:
2393                     continue
2394
2395             if known_idx > last_seq:
2396                 last_segment_url = None
2397                 continue
2398
2399             last_seq += 1
2400
2401             if begin_index < 0 and known_idx < 0:
2402                 # skip from the start when it's negative value
2403                 known_idx = last_seq + begin_index
2404             if lack_early_segments:
2405                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2406             try:
2407                 for idx in range(known_idx, last_seq):
2408                     # do not update sequence here or you'll get skipped some part of it
2409                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2410                     if not should_continue:
2411                         known_idx = idx - 1
2412                         raise ExtractorError('breaking out of outer loop')
2413                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2414                     yield {
2415                         'url': last_segment_url,
2416                         'fragment_count': last_seq,
2417                     }
2418                 if known_idx == last_seq:
2419                     no_fragment_score += 5
2420                 else:
2421                     no_fragment_score = 0
2422                 known_idx = last_seq
2423             except ExtractorError:
2424                 continue
2425
2426             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2427
2428     def _extract_player_url(self, *ytcfgs, webpage=None):
2429         player_url = traverse_obj(
2430             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2431             get_all=False, expected_type=str)
2432         if not player_url:
2433             return
2434         return urljoin('https://www.youtube.com', player_url)
2435
2436     def _download_player_url(self, video_id, fatal=False):
2437         res = self._download_webpage(
2438             'https://www.youtube.com/iframe_api',
2439             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2440         if res:
2441             player_version = self._search_regex(
2442                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2443             if player_version:
2444                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2445
2446     def _signature_cache_id(self, example_sig):
2447         """ Return a string representation of a signature """
2448         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2449
2450     @classmethod
2451     def _extract_player_info(cls, player_url):
2452         for player_re in cls._PLAYER_INFO_RE:
2453             id_m = re.search(player_re, player_url)
2454             if id_m:
2455                 break
2456         else:
2457             raise ExtractorError('Cannot identify player %r' % player_url)
2458         return id_m.group('id')
2459
2460     def _load_player(self, video_id, player_url, fatal=True):
2461         player_id = self._extract_player_info(player_url)
2462         if player_id not in self._code_cache:
2463             code = self._download_webpage(
2464                 player_url, video_id, fatal=fatal,
2465                 note='Downloading player ' + player_id,
2466                 errnote='Download of %s failed' % player_url)
2467             if code:
2468                 self._code_cache[player_id] = code
2469         return self._code_cache.get(player_id)
2470
2471     def _extract_signature_function(self, video_id, player_url, example_sig):
2472         player_id = self._extract_player_info(player_url)
2473
2474         # Read from filesystem cache
2475         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2476         assert os.path.basename(func_id) == func_id
2477
2478         self.write_debug(f'Extracting signature function {func_id}')
2479         cache_spec = self.cache.load('youtube-sigfuncs', func_id)
2480         if cache_spec is not None:
2481             return lambda s: ''.join(s[i] for i in cache_spec)
2482
2483         code = self._load_player(video_id, player_url)
2484         if code:
2485             res = self._parse_sig_js(code)
2486
2487             test_string = ''.join(map(chr, range(len(example_sig))))
2488             cache_res = res(test_string)
2489             cache_spec = [ord(c) for c in cache_res]
2490
2491             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2492             return res
2493
2494     def _print_sig_code(self, func, example_sig):
2495         if not self.get_param('youtube_print_sig_code'):
2496             return
2497
2498         def gen_sig_code(idxs):
2499             def _genslice(start, end, step):
2500                 starts = '' if start == 0 else str(start)
2501                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2502                 steps = '' if step == 1 else (':%d' % step)
2503                 return f's[{starts}{ends}{steps}]'
2504
2505             step = None
2506             # Quelch pyflakes warnings - start will be set when step is set
2507             start = '(Never used)'
2508             for i, prev in zip(idxs[1:], idxs[:-1]):
2509                 if step is not None:
2510                     if i - prev == step:
2511                         continue
2512                     yield _genslice(start, prev, step)
2513                     step = None
2514                     continue
2515                 if i - prev in [-1, 1]:
2516                     step = i - prev
2517                     start = prev
2518                     continue
2519                 else:
2520                     yield 's[%d]' % prev
2521             if step is None:
2522                 yield 's[%d]' % i
2523             else:
2524                 yield _genslice(start, i, step)
2525
2526         test_string = ''.join(map(chr, range(len(example_sig))))
2527         cache_res = func(test_string)
2528         cache_spec = [ord(c) for c in cache_res]
2529         expr_code = ' + '.join(gen_sig_code(cache_spec))
2530         signature_id_tuple = '(%s)' % (
2531             ', '.join(str(len(p)) for p in example_sig.split('.')))
2532         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2533                 '    return %s\n') % (signature_id_tuple, expr_code)
2534         self.to_screen('Extracted signature function:\n' + code)
2535
2536     def _parse_sig_js(self, jscode):
2537         funcname = self._search_regex(
2538             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2539              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2540              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2541              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2542              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2543              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2544              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2545              # Obsolete patterns
2546              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2547              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2548              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2549              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2550              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2551              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2552              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2553              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2554             jscode, 'Initial JS player signature function name', group='sig')
2555
2556         jsi = JSInterpreter(jscode)
2557         initial_function = jsi.extract_function(funcname)
2558         return lambda s: initial_function([s])
2559
2560     def _decrypt_signature(self, s, video_id, player_url):
2561         """Turn the encrypted s field into a working signature"""
2562         try:
2563             player_id = (player_url, self._signature_cache_id(s))
2564             if player_id not in self._player_cache:
2565                 func = self._extract_signature_function(video_id, player_url, s)
2566                 self._player_cache[player_id] = func
2567             func = self._player_cache[player_id]
2568             self._print_sig_code(func, s)
2569             return func(s)
2570         except Exception as e:
2571             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2572
2573     def _decrypt_nsig(self, s, video_id, player_url):
2574         """Turn the encrypted n field into a working signature"""
2575         if player_url is None:
2576             raise ExtractorError('Cannot decrypt nsig without player_url')
2577         player_url = urljoin('https://www.youtube.com', player_url)
2578
2579         sig_id = ('nsig_value', s)
2580         if sig_id in self._player_cache:
2581             return self._player_cache[sig_id]
2582
2583         try:
2584             player_id = ('nsig', player_url)
2585             if player_id not in self._player_cache:
2586                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2587             func = self._player_cache[player_id]
2588             self._player_cache[sig_id] = func(s)
2589             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2590             return self._player_cache[sig_id]
2591         except Exception as e:
2592             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2593
2594     def _extract_n_function_name(self, jscode):
2595         nfunc, idx = self._search_regex(
2596             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2597             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2598         if not idx:
2599             return nfunc
2600         return json.loads(js_to_json(self._search_regex(
2601             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2602             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2603
2604     def _extract_n_function(self, video_id, player_url):
2605         player_id = self._extract_player_info(player_url)
2606         func_code = self.cache.load('youtube-nsig', player_id)
2607
2608         if func_code:
2609             jsi = JSInterpreter(func_code)
2610         else:
2611             jscode = self._load_player(video_id, player_url)
2612             funcname = self._extract_n_function_name(jscode)
2613             jsi = JSInterpreter(jscode)
2614             func_code = jsi.extract_function_code(funcname)
2615             self.cache.store('youtube-nsig', player_id, func_code)
2616
2617         if self.get_param('youtube_print_sig_code'):
2618             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2619
2620         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2621
2622     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2623         """
2624         Extract signatureTimestamp (sts)
2625         Required to tell API what sig/player version is in use.
2626         """
2627         sts = None
2628         if isinstance(ytcfg, dict):
2629             sts = int_or_none(ytcfg.get('STS'))
2630
2631         if not sts:
2632             # Attempt to extract from player
2633             if player_url is None:
2634                 error_msg = 'Cannot extract signature timestamp without player_url.'
2635                 if fatal:
2636                     raise ExtractorError(error_msg)
2637                 self.report_warning(error_msg)
2638                 return
2639             code = self._load_player(video_id, player_url, fatal=fatal)
2640             if code:
2641                 sts = int_or_none(self._search_regex(
2642                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2643                     'JS player signature timestamp', group='sts', fatal=fatal))
2644         return sts
2645
2646     def _mark_watched(self, video_id, player_responses):
2647         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2648             label = 'fully ' if is_full else ''
2649             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2650                             expected_type=url_or_none)
2651             if not url:
2652                 self.report_warning(f'Unable to mark {label}watched')
2653                 return
2654             parsed_url = urllib.parse.urlparse(url)
2655             qs = urllib.parse.parse_qs(parsed_url.query)
2656
2657             # cpn generation algorithm is reverse engineered from base.js.
2658             # In fact it works even with dummy cpn.
2659             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2660             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2661
2662             # # more consistent results setting it to right before the end
2663             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2664
2665             qs.update({
2666                 'ver': ['2'],
2667                 'cpn': [cpn],
2668                 'cmt': video_length,
2669                 'el': 'detailpage',  # otherwise defaults to "shorts"
2670             })
2671
2672             if is_full:
2673                 # these seem to mark watchtime "history" in the real world
2674                 # they're required, so send in a single value
2675                 qs.update({
2676                     'st': video_length,
2677                     'et': video_length,
2678                 })
2679
2680             url = urllib.parse.urlunparse(
2681                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2682
2683             self._download_webpage(
2684                 url, video_id, f'Marking {label}watched',
2685                 'Unable to mark watched', fatal=False)
2686
2687     @staticmethod
2688     def _extract_urls(webpage):
2689         # Embedded YouTube player
2690         entries = [
2691             unescapeHTML(mobj.group('url'))
2692             for mobj in re.finditer(r'''(?x)
2693             (?:
2694                 <iframe[^>]+?src=|
2695                 data-video-url=|
2696                 <embed[^>]+?src=|
2697                 embedSWF\(?:\s*|
2698                 <object[^>]+data=|
2699                 new\s+SWFObject\(
2700             )
2701             (["\'])
2702                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2703                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2704             \1''', webpage)]
2705
2706         # lazyYT YouTube embed
2707         entries.extend(list(map(
2708             unescapeHTML,
2709             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2710
2711         # Wordpress "YouTube Video Importer" plugin
2712         matches = re.findall(r'''(?x)<div[^>]+
2713             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2714             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2715         entries.extend(m[-1] for m in matches)
2716
2717         return entries
2718
2719     @staticmethod
2720     def _extract_url(webpage):
2721         urls = YoutubeIE._extract_urls(webpage)
2722         return urls[0] if urls else None
2723
2724     @classmethod
2725     def extract_id(cls, url):
2726         video_id = cls.get_temp_id(url)
2727         if not video_id:
2728             raise ExtractorError(f'Invalid URL: {url}')
2729         return video_id
2730
2731     def _extract_chapters_from_json(self, data, duration):
2732         chapter_list = traverse_obj(
2733             data, (
2734                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2735                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2736             ), expected_type=list)
2737
2738         return self._extract_chapters(
2739             chapter_list,
2740             chapter_time=lambda chapter: float_or_none(
2741                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2742             chapter_title=lambda chapter: traverse_obj(
2743                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2744             duration=duration)
2745
2746     def _extract_chapters_from_engagement_panel(self, data, duration):
2747         content_list = traverse_obj(
2748             data,
2749             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2750             expected_type=list, default=[])
2751         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2752         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2753
2754         return next(filter(None, (
2755             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2756                                    chapter_time, chapter_title, duration)
2757             for contents in content_list)), [])
2758
2759     def _extract_chapters_from_description(self, description, duration):
2760         return self._extract_chapters(
2761             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
2762             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
2763             duration=duration, strict=False)
2764
2765     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
2766         if not duration:
2767             return
2768         chapter_list = [{
2769             'start_time': chapter_time(chapter),
2770             'title': chapter_title(chapter),
2771         } for chapter in chapter_list or []]
2772         if not strict:
2773             chapter_list.sort(key=lambda c: c['start_time'] or 0)
2774
2775         chapters = [{'start_time': 0}]
2776         for idx, chapter in enumerate(chapter_list):
2777             if chapter['start_time'] is None:
2778                 self.report_warning(f'Incomplete chapter {idx}')
2779             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
2780                 chapters.append(chapter)
2781             else:
2782                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
2783         return chapters[1:]
2784
2785     def _extract_comment(self, comment_renderer, parent=None):
2786         comment_id = comment_renderer.get('commentId')
2787         if not comment_id:
2788             return
2789
2790         text = self._get_text(comment_renderer, 'contentText')
2791
2792         # note: timestamp is an estimate calculated from the current time and time_text
2793         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2794         author = self._get_text(comment_renderer, 'authorText')
2795         author_id = try_get(comment_renderer,
2796                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
2797
2798         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2799                                                        lambda x: x['likeCount']), str)) or 0
2800         author_thumbnail = try_get(comment_renderer,
2801                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
2802
2803         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2804         is_favorited = 'creatorHeart' in (try_get(
2805             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2806         return {
2807             'id': comment_id,
2808             'text': text,
2809             'timestamp': timestamp,
2810             'time_text': time_text,
2811             'like_count': votes,
2812             'is_favorited': is_favorited,
2813             'author': author,
2814             'author_id': author_id,
2815             'author_thumbnail': author_thumbnail,
2816             'author_is_uploader': author_is_uploader,
2817             'parent': parent or 'root'
2818         }
2819
2820     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2821
2822         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2823
2824         def extract_header(contents):
2825             _continuation = None
2826             for content in contents:
2827                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2828                 expected_comment_count = self._get_count(
2829                     comments_header_renderer, 'countText', 'commentsCount')
2830
2831                 if expected_comment_count:
2832                     tracker['est_total'] = expected_comment_count
2833                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2834                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2835
2836                 sort_menu_item = try_get(
2837                     comments_header_renderer,
2838                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2839                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2840
2841                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2842                 if not _continuation:
2843                     continue
2844
2845                 sort_text = str_or_none(sort_menu_item.get('title'))
2846                 if not sort_text:
2847                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2848                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2849                 break
2850             return _continuation
2851
2852         def extract_thread(contents):
2853             if not parent:
2854                 tracker['current_page_thread'] = 0
2855             for content in contents:
2856                 if not parent and tracker['total_parent_comments'] >= max_parents:
2857                     yield
2858                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2859                 comment_renderer = get_first(
2860                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2861                     expected_type=dict, default={})
2862
2863                 comment = self._extract_comment(comment_renderer, parent)
2864                 if not comment:
2865                     continue
2866
2867                 tracker['running_total'] += 1
2868                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2869                 yield comment
2870
2871                 # Attempt to get the replies
2872                 comment_replies_renderer = try_get(
2873                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2874
2875                 if comment_replies_renderer:
2876                     tracker['current_page_thread'] += 1
2877                     comment_entries_iter = self._comment_entries(
2878                         comment_replies_renderer, ytcfg, video_id,
2879                         parent=comment.get('id'), tracker=tracker)
2880                     yield from itertools.islice(comment_entries_iter, min(
2881                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2882
2883         # Keeps track of counts across recursive calls
2884         if not tracker:
2885             tracker = dict(
2886                 running_total=0,
2887                 est_total=0,
2888                 current_page_thread=0,
2889                 total_parent_comments=0,
2890                 total_reply_comments=0)
2891
2892         # TODO: Deprecated
2893         # YouTube comments have a max depth of 2
2894         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2895         if max_depth:
2896             self._downloader.deprecation_warning(
2897                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2898         if max_depth == 1 and parent:
2899             return
2900
2901         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2902             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2903
2904         continuation = self._extract_continuation(root_continuation_data)
2905
2906         response = None
2907         is_forced_continuation = False
2908         is_first_continuation = parent is None
2909         if is_first_continuation and not continuation:
2910             # Sometimes you can get comments by generating the continuation yourself,
2911             # even if YouTube initially reports them being disabled - e.g. stories comments.
2912             # Note: if the comment section is actually disabled, YouTube may return a response with
2913             # required check_get_keys missing. So we will disable that check initially in this case.
2914             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2915             is_forced_continuation = True
2916
2917         for page_num in itertools.count(0):
2918             if not continuation:
2919                 break
2920             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2921             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2922             if page_num == 0:
2923                 if is_first_continuation:
2924                     note_prefix = 'Downloading comment section API JSON'
2925                 else:
2926                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2927                         tracker['current_page_thread'], comment_prog_str)
2928             else:
2929                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2930                     '       ' if parent else '', ' replies' if parent else '',
2931                     page_num, comment_prog_str)
2932
2933             response = self._extract_response(
2934                 item_id=None, query=continuation,
2935                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2936                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2937             is_forced_continuation = False
2938             continuation_contents = traverse_obj(
2939                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2940
2941             continuation = None
2942             for continuation_section in continuation_contents:
2943                 continuation_items = traverse_obj(
2944                     continuation_section,
2945                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2946                     get_all=False, expected_type=list) or []
2947                 if is_first_continuation:
2948                     continuation = extract_header(continuation_items)
2949                     is_first_continuation = False
2950                     if continuation:
2951                         break
2952                     continue
2953
2954                 for entry in extract_thread(continuation_items):
2955                     if not entry:
2956                         return
2957                     yield entry
2958                 continuation = self._extract_continuation({'contents': continuation_items})
2959                 if continuation:
2960                     break
2961
2962         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2963         if message and not parent and tracker['running_total'] == 0:
2964             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2965
2966     @staticmethod
2967     def _generate_comment_continuation(video_id):
2968         """
2969         Generates initial comment section continuation token from given video id
2970         """
2971         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2972         return base64.b64encode(token.encode()).decode()
2973
2974     def _get_comments(self, ytcfg, video_id, contents, webpage):
2975         """Entry for comment extraction"""
2976         def _real_comment_extract(contents):
2977             renderer = next((
2978                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2979                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2980             yield from self._comment_entries(renderer, ytcfg, video_id)
2981
2982         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2983         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2984
2985     @staticmethod
2986     def _get_checkok_params():
2987         return {'contentCheckOk': True, 'racyCheckOk': True}
2988
2989     @classmethod
2990     def _generate_player_context(cls, sts=None):
2991         context = {
2992             'html5Preference': 'HTML5_PREF_WANTS',
2993         }
2994         if sts is not None:
2995             context['signatureTimestamp'] = sts
2996         return {
2997             'playbackContext': {
2998                 'contentPlaybackContext': context
2999             },
3000             **cls._get_checkok_params()
3001         }
3002
3003     @staticmethod
3004     def _is_agegated(player_response):
3005         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3006             return True
3007
3008         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3009         AGE_GATE_REASONS = (
3010             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3011             'age_verification_required', 'age_check_required',  # status
3012         )
3013         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3014
3015     @staticmethod
3016     def _is_unplayable(player_response):
3017         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3018
3019     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
3020
3021         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3022         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3023         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3024         headers = self.generate_api_headers(
3025             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3026
3027         yt_query = {
3028             'videoId': video_id,
3029             'params': '8AEB'  # enable stories
3030         }
3031         yt_query.update(self._generate_player_context(sts))
3032         return self._extract_response(
3033             item_id=video_id, ep='player', query=yt_query,
3034             ytcfg=player_ytcfg, headers=headers, fatal=True,
3035             default_client=client,
3036             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3037         ) or None
3038
3039     def _get_requested_clients(self, url, smuggled_data):
3040         requested_clients = []
3041         default = ['android', 'web']
3042         allowed_clients = sorted(
3043             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3044             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3045         for client in self._configuration_arg('player_client'):
3046             if client in allowed_clients:
3047                 requested_clients.append(client)
3048             elif client == 'default':
3049                 requested_clients.extend(default)
3050             elif client == 'all':
3051                 requested_clients.extend(allowed_clients)
3052             else:
3053                 self.report_warning(f'Skipping unsupported client {client}')
3054         if not requested_clients:
3055             requested_clients = default
3056
3057         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3058             requested_clients.extend(
3059                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3060
3061         return orderedSet(requested_clients)
3062
3063     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3064         initial_pr = None
3065         if webpage:
3066             initial_pr = self._search_json(
3067                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3068
3069         all_clients = set(clients)
3070         clients = clients[::-1]
3071         prs = []
3072
3073         def append_client(*client_names):
3074             """ Append the first client name that exists but not already used """
3075             for client_name in client_names:
3076                 actual_client = _split_innertube_client(client_name)[0]
3077                 if actual_client in INNERTUBE_CLIENTS:
3078                     if actual_client not in all_clients:
3079                         clients.append(client_name)
3080                         all_clients.add(actual_client)
3081                         return
3082
3083         # Android player_response does not have microFormats which are needed for
3084         # extraction of some data. So we return the initial_pr with formats
3085         # stripped out even if not requested by the user
3086         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3087         if initial_pr:
3088             pr = dict(initial_pr)
3089             pr['streamingData'] = None
3090             prs.append(pr)
3091
3092         last_error = None
3093         tried_iframe_fallback = False
3094         player_url = None
3095         while clients:
3096             client, base_client, variant = _split_innertube_client(clients.pop())
3097             player_ytcfg = master_ytcfg if client == 'web' else {}
3098             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3099                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3100
3101             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3102             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3103             if 'js' in self._configuration_arg('player_skip'):
3104                 require_js_player = False
3105                 player_url = None
3106
3107             if not player_url and not tried_iframe_fallback and require_js_player:
3108                 player_url = self._download_player_url(video_id)
3109                 tried_iframe_fallback = True
3110
3111             try:
3112                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3113                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3114             except ExtractorError as e:
3115                 if last_error:
3116                     self.report_warning(last_error)
3117                 last_error = e
3118                 continue
3119
3120             if pr:
3121                 prs.append(pr)
3122
3123             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3124             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3125                 append_client(f'{base_client}_creator')
3126             elif self._is_agegated(pr):
3127                 if variant == 'tv_embedded':
3128                     append_client(f'{base_client}_embedded')
3129                 elif not variant:
3130                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3131
3132         if last_error:
3133             if not len(prs):
3134                 raise last_error
3135             self.report_warning(last_error)
3136         return prs, player_url
3137
3138     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3139         itags, stream_ids = {}, []
3140         itag_qualities, res_qualities = {}, {}
3141         q = qualities([
3142             # Normally tiny is the smallest video-only formats. But
3143             # audio-only formats with unknown quality may get tagged as tiny
3144             'tiny',
3145             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3146             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3147         ])
3148         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3149
3150         for fmt in streaming_formats:
3151             if fmt.get('targetDurationSec'):
3152                 continue
3153
3154             itag = str_or_none(fmt.get('itag'))
3155             audio_track = fmt.get('audioTrack') or {}
3156             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3157             if stream_id in stream_ids:
3158                 continue
3159
3160             quality = fmt.get('quality')
3161             height = int_or_none(fmt.get('height'))
3162             if quality == 'tiny' or not quality:
3163                 quality = fmt.get('audioQuality', '').lower() or quality
3164             # The 3gp format (17) in android client has a quality of "small",
3165             # but is actually worse than other formats
3166             if itag == '17':
3167                 quality = 'tiny'
3168             if quality:
3169                 if itag:
3170                     itag_qualities[itag] = quality
3171                 if height:
3172                     res_qualities[height] = quality
3173             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3174             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3175             # number of fragment that would subsequently requested with (`&sq=N`)
3176             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3177                 continue
3178
3179             fmt_url = fmt.get('url')
3180             if not fmt_url:
3181                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3182                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3183                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3184                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3185                     continue
3186                 try:
3187                     fmt_url += '&%s=%s' % (
3188                         traverse_obj(sc, ('sp', -1)) or 'signature',
3189                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3190                     )
3191                 except ExtractorError as e:
3192                     self.report_warning('Signature extraction failed: Some formats may be missing', only_once=True)
3193                     self.write_debug(e, only_once=True)
3194                     continue
3195
3196             query = parse_qs(fmt_url)
3197             throttled = False
3198             if query.get('n'):
3199                 try:
3200                     fmt_url = update_url_query(fmt_url, {
3201                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3202                 except ExtractorError as e:
3203                     self.report_warning(
3204                         'nsig extraction failed: You may experience throttling for some formats\n'
3205                         f'n = {query["n"][0]} ; player = {player_url}', only_once=True)
3206                     self.write_debug(e, only_once=True)
3207                     throttled = True
3208
3209             if itag:
3210                 itags[itag] = 'https'
3211                 stream_ids.append(stream_id)
3212
3213             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3214             language_preference = (
3215                 10 if audio_track.get('audioIsDefault') and 10
3216                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3217                 else -1)
3218             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3219             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3220             # Make sure to avoid false positives with small duration differences.
3221             # Eg: __2ABJjxzNo, ySuUZEjARPY
3222             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3223             if is_damaged:
3224                 self.report_warning(
3225                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3226             dct = {
3227                 'asr': int_or_none(fmt.get('audioSampleRate')),
3228                 'filesize': int_or_none(fmt.get('contentLength')),
3229                 'format_id': itag,
3230                 'format_note': join_nonempty(
3231                     '%s%s' % (audio_track.get('displayName') or '',
3232                               ' (default)' if language_preference > 0 else ''),
3233                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3234                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3235                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3236                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3237                 'fps': int_or_none(fmt.get('fps')) or None,
3238                 'height': height,
3239                 'quality': q(quality),
3240                 'has_drm': bool(fmt.get('drmFamilies')),
3241                 'tbr': tbr,
3242                 'url': fmt_url,
3243                 'width': int_or_none(fmt.get('width')),
3244                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3245                                           'desc' if language_preference < -1 else ''),
3246                 'language_preference': language_preference,
3247                 # Strictly de-prioritize damaged and 3gp formats
3248                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3249             }
3250             mime_mobj = re.match(
3251                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3252             if mime_mobj:
3253                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3254                 dct.update(parse_codecs(mime_mobj.group(2)))
3255             no_audio = dct.get('acodec') == 'none'
3256             no_video = dct.get('vcodec') == 'none'
3257             if no_audio:
3258                 dct['vbr'] = tbr
3259             if no_video:
3260                 dct['abr'] = tbr
3261             if no_audio or no_video:
3262                 dct['downloader_options'] = {
3263                     # Youtube throttles chunks >~10M
3264                     'http_chunk_size': 10485760,
3265                 }
3266                 if dct.get('ext'):
3267                     dct['container'] = dct['ext'] + '_dash'
3268             yield dct
3269
3270         live_from_start = is_live and self.get_param('live_from_start')
3271         skip_manifests = self._configuration_arg('skip')
3272         if not self.get_param('youtube_include_hls_manifest', True):
3273             skip_manifests.append('hls')
3274         if not self.get_param('youtube_include_dash_manifest', True):
3275             skip_manifests.append('dash')
3276         get_dash = 'dash' not in skip_manifests and (
3277             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3278         get_hls = not live_from_start and 'hls' not in skip_manifests
3279
3280         def process_manifest_format(f, proto, itag):
3281             if itag in itags:
3282                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3283                     return False
3284                 itag = f'{itag}-{proto}'
3285             if itag:
3286                 f['format_id'] = itag
3287                 itags[itag] = proto
3288
3289             f['quality'] = next((
3290                 q(qdict[val])
3291                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3292                 if val in qdict), -1)
3293             return True
3294
3295         for sd in streaming_data:
3296             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3297             if hls_manifest_url:
3298                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3299                     if process_manifest_format(f, 'hls', self._search_regex(
3300                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3301                         yield f
3302
3303             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3304             if dash_manifest_url:
3305                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3306                     if process_manifest_format(f, 'dash', f['format_id']):
3307                         f['filesize'] = int_or_none(self._search_regex(
3308                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3309                         if live_from_start:
3310                             f['is_from_start'] = True
3311
3312                         yield f
3313
3314     def _extract_storyboard(self, player_responses, duration):
3315         spec = get_first(
3316             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3317         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3318         if not base_url:
3319             return
3320         L = len(spec) - 1
3321         for i, args in enumerate(spec):
3322             args = args.split('#')
3323             counts = list(map(int_or_none, args[:5]))
3324             if len(args) != 8 or not all(counts):
3325                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3326                 continue
3327             width, height, frame_count, cols, rows = counts
3328             N, sigh = args[6:]
3329
3330             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3331             fragment_count = frame_count / (cols * rows)
3332             fragment_duration = duration / fragment_count
3333             yield {
3334                 'format_id': f'sb{i}',
3335                 'format_note': 'storyboard',
3336                 'ext': 'mhtml',
3337                 'protocol': 'mhtml',
3338                 'acodec': 'none',
3339                 'vcodec': 'none',
3340                 'url': url,
3341                 'width': width,
3342                 'height': height,
3343                 'fps': frame_count / duration,
3344                 'rows': rows,
3345                 'columns': cols,
3346                 'fragments': [{
3347                     'url': url.replace('$M', str(j)),
3348                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3349                 } for j in range(math.ceil(fragment_count))],
3350             }
3351
3352     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3353         webpage = None
3354         if 'webpage' not in self._configuration_arg('player_skip'):
3355             webpage = self._download_webpage(
3356                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3357
3358         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3359
3360         player_responses, player_url = self._extract_player_responses(
3361             self._get_requested_clients(url, smuggled_data),
3362             video_id, webpage, master_ytcfg)
3363
3364         return webpage, master_ytcfg, player_responses, player_url
3365
3366     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3367         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3368         is_live = get_first(video_details, 'isLive')
3369         if is_live is None:
3370             is_live = get_first(live_broadcast_details, 'isLiveNow')
3371
3372         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3373         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3374
3375         return live_broadcast_details, is_live, streaming_data, formats
3376
3377     def _real_extract(self, url):
3378         url, smuggled_data = unsmuggle_url(url, {})
3379         video_id = self._match_id(url)
3380
3381         base_url = self.http_scheme() + '//www.youtube.com/'
3382         webpage_url = base_url + 'watch?v=' + video_id
3383
3384         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3385
3386         playability_statuses = traverse_obj(
3387             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3388
3389         trailer_video_id = get_first(
3390             playability_statuses,
3391             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3392             expected_type=str)
3393         if trailer_video_id:
3394             return self.url_result(
3395                 trailer_video_id, self.ie_key(), trailer_video_id)
3396
3397         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3398                        if webpage else (lambda x: None))
3399
3400         video_details = traverse_obj(
3401             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3402         microformats = traverse_obj(
3403             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3404             expected_type=dict, default=[])
3405         video_title = (
3406             get_first(video_details, 'title')
3407             or self._get_text(microformats, (..., 'title'))
3408             or search_meta(['og:title', 'twitter:title', 'title']))
3409         video_description = get_first(video_details, 'shortDescription')
3410
3411         multifeed_metadata_list = get_first(
3412             player_responses,
3413             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3414             expected_type=str)
3415         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3416             if self.get_param('noplaylist'):
3417                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3418             else:
3419                 entries = []
3420                 feed_ids = []
3421                 for feed in multifeed_metadata_list.split(','):
3422                     # Unquote should take place before split on comma (,) since textual
3423                     # fields may contain comma as well (see
3424                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3425                     feed_data = urllib.parse.parse_qs(
3426                         urllib.parse.unquote_plus(feed))
3427
3428                     def feed_entry(name):
3429                         return try_get(
3430                             feed_data, lambda x: x[name][0], str)
3431
3432                     feed_id = feed_entry('id')
3433                     if not feed_id:
3434                         continue
3435                     feed_title = feed_entry('title')
3436                     title = video_title
3437                     if feed_title:
3438                         title += ' (%s)' % feed_title
3439                     entries.append({
3440                         '_type': 'url_transparent',
3441                         'ie_key': 'Youtube',
3442                         'url': smuggle_url(
3443                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3444                             {'force_singlefeed': True}),
3445                         'title': title,
3446                     })
3447                     feed_ids.append(feed_id)
3448                 self.to_screen(
3449                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3450                     % (', '.join(feed_ids), video_id))
3451                 return self.playlist_result(
3452                     entries, video_id, video_title, video_description)
3453
3454         duration = int_or_none(
3455             get_first(video_details, 'lengthSeconds')
3456             or get_first(microformats, 'lengthSeconds')
3457             or parse_duration(search_meta('duration'))) or None
3458
3459         if get_first(video_details, 'isPostLiveDvr'):
3460             self.write_debug('Video is in Post-Live Manifestless mode')
3461             if (duration or 0) > 4 * 3600:
3462                 self.report_warning(
3463                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3464                     'This is a known issue and patches are welcome')
3465
3466         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3467             video_id, microformats, video_details, player_responses, player_url, duration)
3468
3469         if not formats:
3470             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3471                 self.report_drm(video_id)
3472             pemr = get_first(
3473                 playability_statuses,
3474                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3475             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3476             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3477             if subreason:
3478                 if subreason == 'The uploader has not made this video available in your country.':
3479                     countries = get_first(microformats, 'availableCountries')
3480                     if not countries:
3481                         regions_allowed = search_meta('regionsAllowed')
3482                         countries = regions_allowed.split(',') if regions_allowed else None
3483                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3484                 reason += f'. {subreason}'
3485             if reason:
3486                 self.raise_no_formats(reason, expected=True)
3487
3488         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3489         if not keywords and webpage:
3490             keywords = [
3491                 unescapeHTML(m.group('content'))
3492                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3493         for keyword in keywords:
3494             if keyword.startswith('yt:stretch='):
3495                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3496                 if mobj:
3497                     # NB: float is intentional for forcing float division
3498                     w, h = (float(v) for v in mobj.groups())
3499                     if w > 0 and h > 0:
3500                         ratio = w / h
3501                         for f in formats:
3502                             if f.get('vcodec') != 'none':
3503                                 f['stretched_ratio'] = ratio
3504                         break
3505         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3506         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3507         if thumbnail_url:
3508             thumbnails.append({
3509                 'url': thumbnail_url,
3510             })
3511         original_thumbnails = thumbnails.copy()
3512
3513         # The best resolution thumbnails sometimes does not appear in the webpage
3514         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3515         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3516         thumbnail_names = [
3517             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3518             # in resolution, these are not the custom thumbnail. So de-prioritize them
3519             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3520             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3521         ]
3522         n_thumbnail_names = len(thumbnail_names)
3523         thumbnails.extend({
3524             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3525                 video_id=video_id, name=name, ext=ext,
3526                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3527         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3528         for thumb in thumbnails:
3529             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3530             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3531         self._remove_duplicate_formats(thumbnails)
3532         self._downloader._sort_thumbnails(original_thumbnails)
3533
3534         category = get_first(microformats, 'category') or search_meta('genre')
3535         channel_id = str_or_none(
3536             get_first(video_details, 'channelId')
3537             or get_first(microformats, 'externalChannelId')
3538             or search_meta('channelId'))
3539         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3540
3541         live_content = get_first(video_details, 'isLiveContent')
3542         is_upcoming = get_first(video_details, 'isUpcoming')
3543         if is_live is None:
3544             if is_upcoming or live_content is False:
3545                 is_live = False
3546         if is_upcoming is None and (live_content or is_live):
3547             is_upcoming = False
3548         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3549         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3550         if not duration and live_end_time and live_start_time:
3551             duration = live_end_time - live_start_time
3552
3553         if is_live and self.get_param('live_from_start'):
3554             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3555
3556         formats.extend(self._extract_storyboard(player_responses, duration))
3557
3558         # Source is given priority since formats that throttle are given lower source_preference
3559         # When throttling issue is fully fixed, remove this
3560         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3561
3562         info = {
3563             'id': video_id,
3564             'title': video_title,
3565             'formats': formats,
3566             'thumbnails': thumbnails,
3567             # The best thumbnail that we are sure exists. Prevents unnecessary
3568             # URL checking if user don't care about getting the best possible thumbnail
3569             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3570             'description': video_description,
3571             'uploader': get_first(video_details, 'author'),
3572             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3573             'uploader_url': owner_profile_url,
3574             'channel_id': channel_id,
3575             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3576             'duration': duration,
3577             'view_count': int_or_none(
3578                 get_first((video_details, microformats), (..., 'viewCount'))
3579                 or search_meta('interactionCount')),
3580             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3581             'age_limit': 18 if (
3582                 get_first(microformats, 'isFamilySafe') is False
3583                 or search_meta('isFamilyFriendly') == 'false'
3584                 or search_meta('og:restrictions:age') == '18+') else 0,
3585             'webpage_url': webpage_url,
3586             'categories': [category] if category else None,
3587             'tags': keywords,
3588             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3589             'is_live': is_live,
3590             'was_live': (False if is_live or is_upcoming or live_content is False
3591                          else None if is_live is None or is_upcoming is None
3592                          else live_content),
3593             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3594             'release_timestamp': live_start_time,
3595         }
3596
3597         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3598         if pctr:
3599             def get_lang_code(track):
3600                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3601                         or track.get('languageCode'))
3602
3603             # Converted into dicts to remove duplicates
3604             captions = {
3605                 get_lang_code(sub): sub
3606                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3607             translation_languages = {
3608                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3609                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3610
3611             def process_language(container, base_url, lang_code, sub_name, query):
3612                 lang_subs = container.setdefault(lang_code, [])
3613                 for fmt in self._SUBTITLE_FORMATS:
3614                     query.update({
3615                         'fmt': fmt,
3616                     })
3617                     lang_subs.append({
3618                         'ext': fmt,
3619                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3620                         'name': sub_name,
3621                     })
3622
3623             subtitles, automatic_captions = {}, {}
3624             for lang_code, caption_track in captions.items():
3625                 base_url = caption_track.get('baseUrl')
3626                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3627                 if not base_url:
3628                     continue
3629                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3630                 if caption_track.get('kind') != 'asr':
3631                     if not lang_code:
3632                         continue
3633                     process_language(
3634                         subtitles, base_url, lang_code, lang_name, {})
3635                     if not caption_track.get('isTranslatable'):
3636                         continue
3637                 for trans_code, trans_name in translation_languages.items():
3638                     if not trans_code:
3639                         continue
3640                     orig_trans_code = trans_code
3641                     if caption_track.get('kind') != 'asr':
3642                         if 'translated_subs' in self._configuration_arg('skip'):
3643                             continue
3644                         trans_code += f'-{lang_code}'
3645                         trans_name += format_field(lang_name, None, ' from %s')
3646                     # Add an "-orig" label to the original language so that it can be distinguished.
3647                     # The subs are returned without "-orig" as well for compatibility
3648                     if lang_code == f'a-{orig_trans_code}':
3649                         process_language(
3650                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3651                     # Setting tlang=lang returns damaged subtitles.
3652                     process_language(automatic_captions, base_url, trans_code, trans_name,
3653                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3654             info['automatic_captions'] = automatic_captions
3655             info['subtitles'] = subtitles
3656
3657         parsed_url = urllib.parse.urlparse(url)
3658         for component in [parsed_url.fragment, parsed_url.query]:
3659             query = urllib.parse.parse_qs(component)
3660             for k, v in query.items():
3661                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3662                     d_k += '_time'
3663                     if d_k not in info and k in s_ks:
3664                         info[d_k] = parse_duration(query[k][0])
3665
3666         # Youtube Music Auto-generated description
3667         if video_description:
3668             mobj = re.search(
3669                 r'''(?xs)
3670                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3671                     (?P<album>[^\n]+)
3672                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3673                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3674                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3675                     .+\nAuto-generated\ by\ YouTube\.\s*$
3676                 ''', video_description)
3677             if mobj:
3678                 release_year = mobj.group('release_year')
3679                 release_date = mobj.group('release_date')
3680                 if release_date:
3681                     release_date = release_date.replace('-', '')
3682                     if not release_year:
3683                         release_year = release_date[:4]
3684                 info.update({
3685                     'album': mobj.group('album'.strip()),
3686                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3687                     'track': mobj.group('track').strip(),
3688                     'release_date': release_date,
3689                     'release_year': int_or_none(release_year),
3690                 })
3691
3692         initial_data = None
3693         if webpage:
3694             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
3695         if not initial_data:
3696             query = {'videoId': video_id}
3697             query.update(self._get_checkok_params())
3698             initial_data = self._extract_response(
3699                 item_id=video_id, ep='next', fatal=False,
3700                 ytcfg=master_ytcfg, query=query,
3701                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3702                 note='Downloading initial data API JSON')
3703
3704         info['comment_count'] = traverse_obj(initial_data, (
3705             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
3706             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
3707         ), (
3708             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
3709             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
3710         ), expected_type=int_or_none, get_all=False)
3711
3712         try:  # This will error if there is no livechat
3713             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3714         except (KeyError, IndexError, TypeError):
3715             pass
3716         else:
3717             info.setdefault('subtitles', {})['live_chat'] = [{
3718                 # url is needed to set cookies
3719                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
3720                 'video_id': video_id,
3721                 'ext': 'json',
3722                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3723             }]
3724
3725         if initial_data:
3726             info['chapters'] = (
3727                 self._extract_chapters_from_json(initial_data, duration)
3728                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3729                 or self._extract_chapters_from_description(video_description, duration)
3730                 or None)
3731
3732         contents = traverse_obj(
3733             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3734             expected_type=list, default=[])
3735
3736         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3737         if vpir:
3738             stl = vpir.get('superTitleLink')
3739             if stl:
3740                 stl = self._get_text(stl)
3741                 if try_get(
3742                         vpir,
3743                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3744                     info['location'] = stl
3745                 else:
3746                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3747                     if mobj:
3748                         info.update({
3749                             'series': mobj.group(1),
3750                             'season_number': int(mobj.group(2)),
3751                             'episode_number': int(mobj.group(3)),
3752                         })
3753             for tlb in (try_get(
3754                     vpir,
3755                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3756                     list) or []):
3757                 tbr = tlb.get('toggleButtonRenderer') or {}
3758                 for getter, regex in [(
3759                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3760                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3761                             lambda x: x['accessibility'],
3762                             lambda x: x['accessibilityData']['accessibilityData'],
3763                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3764                     label = (try_get(tbr, getter, dict) or {}).get('label')
3765                     if label:
3766                         mobj = re.match(regex, label)
3767                         if mobj:
3768                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3769                             break
3770             sbr_tooltip = try_get(
3771                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3772             if sbr_tooltip:
3773                 like_count, dislike_count = sbr_tooltip.split(' / ')
3774                 info.update({
3775                     'like_count': str_to_int(like_count),
3776                     'dislike_count': str_to_int(dislike_count),
3777                 })
3778         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3779         if vsir:
3780             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3781             info.update({
3782                 'channel': self._get_text(vor, 'title'),
3783                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3784
3785             rows = try_get(
3786                 vsir,
3787                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3788                 list) or []
3789             multiple_songs = False
3790             for row in rows:
3791                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3792                     multiple_songs = True
3793                     break
3794             for row in rows:
3795                 mrr = row.get('metadataRowRenderer') or {}
3796                 mrr_title = mrr.get('title')
3797                 if not mrr_title:
3798                     continue
3799                 mrr_title = self._get_text(mrr, 'title')
3800                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3801                 if mrr_title == 'License':
3802                     info['license'] = mrr_contents_text
3803                 elif not multiple_songs:
3804                     if mrr_title == 'Album':
3805                         info['album'] = mrr_contents_text
3806                     elif mrr_title == 'Artist':
3807                         info['artist'] = mrr_contents_text
3808                     elif mrr_title == 'Song':
3809                         info['track'] = mrr_contents_text
3810
3811         fallbacks = {
3812             'channel': 'uploader',
3813             'channel_id': 'uploader_id',
3814             'channel_url': 'uploader_url',
3815         }
3816
3817         # The upload date for scheduled, live and past live streams / premieres in microformats
3818         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3819         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3820         upload_date = (
3821             unified_strdate(get_first(microformats, 'uploadDate'))
3822             or unified_strdate(search_meta('uploadDate')))
3823         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3824             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3825         info['upload_date'] = upload_date
3826
3827         for to, frm in fallbacks.items():
3828             if not info.get(to):
3829                 info[to] = info.get(frm)
3830
3831         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3832             v = info.get(s_k)
3833             if v:
3834                 info[d_k] = v
3835
3836         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3837         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3838         is_membersonly = None
3839         is_premium = None
3840         if initial_data and is_private is not None:
3841             is_membersonly = False
3842             is_premium = False
3843             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3844             badge_labels = set()
3845             for content in contents:
3846                 if not isinstance(content, dict):
3847                     continue
3848                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3849             for badge_label in badge_labels:
3850                 if badge_label.lower() == 'members only':
3851                     is_membersonly = True
3852                 elif badge_label.lower() == 'premium':
3853                     is_premium = True
3854                 elif badge_label.lower() == 'unlisted':
3855                     is_unlisted = True
3856
3857         info['availability'] = self._availability(
3858             is_private=is_private,
3859             needs_premium=is_premium,
3860             needs_subscription=is_membersonly,
3861             needs_auth=info['age_limit'] >= 18,
3862             is_unlisted=None if is_private is None else is_unlisted)
3863
3864         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3865
3866         self.mark_watched(video_id, player_responses)
3867
3868         return info
3869
3870
3871 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3872
3873     @staticmethod
3874     def passthrough_smuggled_data(func):
3875         def _smuggle(entries, smuggled_data):
3876             for entry in entries:
3877                 # TODO: Convert URL to music.youtube instead.
3878                 # Do we need to passthrough any other smuggled_data?
3879                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3880                 yield entry
3881
3882         @functools.wraps(func)
3883         def wrapper(self, url):
3884             url, smuggled_data = unsmuggle_url(url, {})
3885             if self.is_music_url(url):
3886                 smuggled_data['is_music_url'] = True
3887             info_dict = func(self, url, smuggled_data)
3888             if smuggled_data and info_dict.get('entries'):
3889                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3890             return info_dict
3891         return wrapper
3892
3893     def _extract_channel_id(self, webpage):
3894         channel_id = self._html_search_meta(
3895             'channelId', webpage, 'channel id', default=None)
3896         if channel_id:
3897             return channel_id
3898         channel_url = self._html_search_meta(
3899             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3900              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3901              'twitter:app:url:googleplay'), webpage, 'channel url')
3902         return self._search_regex(
3903             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3904             channel_url, 'channel id')
3905
3906     @staticmethod
3907     def _extract_basic_item_renderer(item):
3908         # Modified from _extract_grid_item_renderer
3909         known_basic_renderers = (
3910             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3911         )
3912         for key, renderer in item.items():
3913             if not isinstance(renderer, dict):
3914                 continue
3915             elif key in known_basic_renderers:
3916                 return renderer
3917             elif key.startswith('grid') and key.endswith('Renderer'):
3918                 return renderer
3919
3920     def _grid_entries(self, grid_renderer):
3921         for item in grid_renderer['items']:
3922             if not isinstance(item, dict):
3923                 continue
3924             renderer = self._extract_basic_item_renderer(item)
3925             if not isinstance(renderer, dict):
3926                 continue
3927             title = self._get_text(renderer, 'title')
3928
3929             # playlist
3930             playlist_id = renderer.get('playlistId')
3931             if playlist_id:
3932                 yield self.url_result(
3933                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3934                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3935                     video_title=title)
3936                 continue
3937             # video
3938             video_id = renderer.get('videoId')
3939             if video_id:
3940                 yield self._extract_video(renderer)
3941                 continue
3942             # channel
3943             channel_id = renderer.get('channelId')
3944             if channel_id:
3945                 yield self.url_result(
3946                     'https://www.youtube.com/channel/%s' % channel_id,
3947                     ie=YoutubeTabIE.ie_key(), video_title=title)
3948                 continue
3949             # generic endpoint URL support
3950             ep_url = urljoin('https://www.youtube.com/', try_get(
3951                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3952                 str))
3953             if ep_url:
3954                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3955                     if ie.suitable(ep_url):
3956                         yield self.url_result(
3957                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3958                         break
3959
3960     def _music_reponsive_list_entry(self, renderer):
3961         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3962         if video_id:
3963             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3964                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3965         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3966         if playlist_id:
3967             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3968             if video_id:
3969                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3970                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3971             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3972                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3973         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3974         if browse_id:
3975             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3976                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3977
3978     def _shelf_entries_from_content(self, shelf_renderer):
3979         content = shelf_renderer.get('content')
3980         if not isinstance(content, dict):
3981             return
3982         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3983         if renderer:
3984             # TODO: add support for nested playlists so each shelf is processed
3985             # as separate playlist
3986             # TODO: this includes only first N items
3987             yield from self._grid_entries(renderer)
3988         renderer = content.get('horizontalListRenderer')
3989         if renderer:
3990             # TODO
3991             pass
3992
3993     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3994         ep = try_get(
3995             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3996             str)
3997         shelf_url = urljoin('https://www.youtube.com', ep)
3998         if shelf_url:
3999             # Skipping links to another channels, note that checking for
4000             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4001             # will not work
4002             if skip_channels and '/channels?' in shelf_url:
4003                 return
4004             title = self._get_text(shelf_renderer, 'title')
4005             yield self.url_result(shelf_url, video_title=title)
4006         # Shelf may not contain shelf URL, fallback to extraction from content
4007         yield from self._shelf_entries_from_content(shelf_renderer)
4008
4009     def _playlist_entries(self, video_list_renderer):
4010         for content in video_list_renderer['contents']:
4011             if not isinstance(content, dict):
4012                 continue
4013             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4014             if not isinstance(renderer, dict):
4015                 continue
4016             video_id = renderer.get('videoId')
4017             if not video_id:
4018                 continue
4019             yield self._extract_video(renderer)
4020
4021     def _rich_entries(self, rich_grid_renderer):
4022         renderer = try_get(
4023             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
4024         video_id = renderer.get('videoId')
4025         if not video_id:
4026             return
4027         yield self._extract_video(renderer)
4028
4029     def _video_entry(self, video_renderer):
4030         video_id = video_renderer.get('videoId')
4031         if video_id:
4032             return self._extract_video(video_renderer)
4033
4034     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4035         url = urljoin('https://youtube.com', traverse_obj(
4036             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4037         if url:
4038             return self.url_result(
4039                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4040
4041     def _post_thread_entries(self, post_thread_renderer):
4042         post_renderer = try_get(
4043             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4044         if not post_renderer:
4045             return
4046         # video attachment
4047         video_renderer = try_get(
4048             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4049         video_id = video_renderer.get('videoId')
4050         if video_id:
4051             entry = self._extract_video(video_renderer)
4052             if entry:
4053                 yield entry
4054         # playlist attachment
4055         playlist_id = try_get(
4056             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4057         if playlist_id:
4058             yield self.url_result(
4059                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4060                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4061         # inline video links
4062         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4063         for run in runs:
4064             if not isinstance(run, dict):
4065                 continue
4066             ep_url = try_get(
4067                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4068             if not ep_url:
4069                 continue
4070             if not YoutubeIE.suitable(ep_url):
4071                 continue
4072             ep_video_id = YoutubeIE._match_id(ep_url)
4073             if video_id == ep_video_id:
4074                 continue
4075             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4076
4077     def _post_thread_continuation_entries(self, post_thread_continuation):
4078         contents = post_thread_continuation.get('contents')
4079         if not isinstance(contents, list):
4080             return
4081         for content in contents:
4082             renderer = content.get('backstagePostThreadRenderer')
4083             if isinstance(renderer, dict):
4084                 yield from self._post_thread_entries(renderer)
4085                 continue
4086             renderer = content.get('videoRenderer')
4087             if isinstance(renderer, dict):
4088                 yield self._video_entry(renderer)
4089
4090     r''' # unused
4091     def _rich_grid_entries(self, contents):
4092         for content in contents:
4093             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4094             if video_renderer:
4095                 entry = self._video_entry(video_renderer)
4096                 if entry:
4097                     yield entry
4098     '''
4099
4100     def _extract_entries(self, parent_renderer, continuation_list):
4101         # continuation_list is modified in-place with continuation_list = [continuation_token]
4102         continuation_list[:] = [None]
4103         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4104         for content in contents:
4105             if not isinstance(content, dict):
4106                 continue
4107             is_renderer = traverse_obj(
4108                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4109                 expected_type=dict)
4110             if not is_renderer:
4111                 renderer = content.get('richItemRenderer')
4112                 if renderer:
4113                     for entry in self._rich_entries(renderer):
4114                         yield entry
4115                     continuation_list[0] = self._extract_continuation(parent_renderer)
4116                 continue
4117             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4118             for isr_content in isr_contents:
4119                 if not isinstance(isr_content, dict):
4120                     continue
4121
4122                 known_renderers = {
4123                     'playlistVideoListRenderer': self._playlist_entries,
4124                     'gridRenderer': self._grid_entries,
4125                     'reelShelfRenderer': self._grid_entries,
4126                     'shelfRenderer': self._shelf_entries,
4127                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4128                     'backstagePostThreadRenderer': self._post_thread_entries,
4129                     'videoRenderer': lambda x: [self._video_entry(x)],
4130                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4131                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4132                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4133                 }
4134                 for key, renderer in isr_content.items():
4135                     if key not in known_renderers:
4136                         continue
4137                     for entry in known_renderers[key](renderer):
4138                         if entry:
4139                             yield entry
4140                     continuation_list[0] = self._extract_continuation(renderer)
4141                     break
4142
4143             if not continuation_list[0]:
4144                 continuation_list[0] = self._extract_continuation(is_renderer)
4145
4146         if not continuation_list[0]:
4147             continuation_list[0] = self._extract_continuation(parent_renderer)
4148
4149     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4150         continuation_list = [None]
4151         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4152         tab_content = try_get(tab, lambda x: x['content'], dict)
4153         if not tab_content:
4154             return
4155         parent_renderer = (
4156             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4157             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4158         yield from extract_entries(parent_renderer)
4159         continuation = continuation_list[0]
4160
4161         for page_num in itertools.count(1):
4162             if not continuation:
4163                 break
4164             headers = self.generate_api_headers(
4165                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4166             response = self._extract_response(
4167                 item_id=f'{item_id} page {page_num}',
4168                 query=continuation, headers=headers, ytcfg=ytcfg,
4169                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4170
4171             if not response:
4172                 break
4173             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4174             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4175             visitor_data = self._extract_visitor_data(response) or visitor_data
4176
4177             known_continuation_renderers = {
4178                 'playlistVideoListContinuation': self._playlist_entries,
4179                 'gridContinuation': self._grid_entries,
4180                 'itemSectionContinuation': self._post_thread_continuation_entries,
4181                 'sectionListContinuation': extract_entries,  # for feeds
4182             }
4183             continuation_contents = try_get(
4184                 response, lambda x: x['continuationContents'], dict) or {}
4185             continuation_renderer = None
4186             for key, value in continuation_contents.items():
4187                 if key not in known_continuation_renderers:
4188                     continue
4189                 continuation_renderer = value
4190                 continuation_list = [None]
4191                 yield from known_continuation_renderers[key](continuation_renderer)
4192                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4193                 break
4194             if continuation_renderer:
4195                 continue
4196
4197             known_renderers = {
4198                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4199                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4200                 'gridVideoRenderer': (self._grid_entries, 'items'),
4201                 'gridChannelRenderer': (self._grid_entries, 'items'),
4202                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4203                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4204                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4205                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4206             }
4207             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4208             continuation_items = try_get(
4209                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4210             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4211             video_items_renderer = None
4212             for key, value in continuation_item.items():
4213                 if key not in known_renderers:
4214                     continue
4215                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4216                 continuation_list = [None]
4217                 yield from known_renderers[key][0](video_items_renderer)
4218                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4219                 break
4220             if video_items_renderer:
4221                 continue
4222             break
4223
4224     @staticmethod
4225     def _extract_selected_tab(tabs, fatal=True):
4226         for tab in tabs:
4227             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4228             if renderer.get('selected') is True:
4229                 return renderer
4230         else:
4231             if fatal:
4232                 raise ExtractorError('Unable to find selected tab')
4233
4234     def _extract_uploader(self, data):
4235         uploader = {}
4236         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4237         owner = try_get(
4238             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4239         if owner:
4240             owner_text = owner.get('text')
4241             uploader['uploader'] = self._search_regex(
4242                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4243             uploader['uploader_id'] = try_get(
4244                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], str)
4245             uploader['uploader_url'] = urljoin(
4246                 'https://www.youtube.com/',
4247                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], str))
4248         return {k: v for k, v in uploader.items() if v is not None}
4249
4250     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4251         playlist_id = title = description = channel_url = channel_name = channel_id = None
4252         tags = []
4253
4254         selected_tab = self._extract_selected_tab(tabs)
4255         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4256         renderer = try_get(
4257             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4258         if renderer:
4259             channel_name = renderer.get('title')
4260             channel_url = renderer.get('channelUrl')
4261             channel_id = renderer.get('externalId')
4262         else:
4263             renderer = try_get(
4264                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4265
4266         if renderer:
4267             title = renderer.get('title')
4268             description = renderer.get('description', '')
4269             playlist_id = channel_id
4270             tags = renderer.get('keywords', '').split()
4271
4272         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4273         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4274         def _get_uncropped(url):
4275             return url_or_none((url or '').split('=')[0] + '=s0')
4276
4277         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4278         if avatar_thumbnails:
4279             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4280             if uncropped_avatar:
4281                 avatar_thumbnails.append({
4282                     'url': uncropped_avatar,
4283                     'id': 'avatar_uncropped',
4284                     'preference': 1
4285                 })
4286
4287         channel_banners = self._extract_thumbnails(
4288             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4289         for banner in channel_banners:
4290             banner['preference'] = -10
4291
4292         if channel_banners:
4293             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4294             if uncropped_banner:
4295                 channel_banners.append({
4296                     'url': uncropped_banner,
4297                     'id': 'banner_uncropped',
4298                     'preference': -5
4299                 })
4300
4301         primary_thumbnails = self._extract_thumbnails(
4302             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4303
4304         if playlist_id is None:
4305             playlist_id = item_id
4306
4307         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4308         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4309         if title is None:
4310             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4311         title += format_field(selected_tab, 'title', ' - %s')
4312         title += format_field(selected_tab, 'expandedText', ' - %s')
4313
4314         metadata = {
4315             'playlist_id': playlist_id,
4316             'playlist_title': title,
4317             'playlist_description': description,
4318             'uploader': channel_name,
4319             'uploader_id': channel_id,
4320             'uploader_url': channel_url,
4321             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4322             'tags': tags,
4323             'view_count': self._get_count(playlist_stats, 1),
4324             'availability': self._extract_availability(data),
4325             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4326             'playlist_count': self._get_count(playlist_stats, 0),
4327             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4328         }
4329         if not channel_id:
4330             metadata.update(self._extract_uploader(data))
4331         metadata.update({
4332             'channel': metadata['uploader'],
4333             'channel_id': metadata['uploader_id'],
4334             'channel_url': metadata['uploader_url']})
4335         return self.playlist_result(
4336             self._entries(
4337                 selected_tab, playlist_id, ytcfg,
4338                 self._extract_account_syncid(ytcfg, data),
4339                 self._extract_visitor_data(data, ytcfg)),
4340             **metadata)
4341
4342     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4343         first_id = last_id = response = None
4344         for page_num in itertools.count(1):
4345             videos = list(self._playlist_entries(playlist))
4346             if not videos:
4347                 return
4348             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4349             if start >= len(videos):
4350                 return
4351             yield from videos[start:]
4352             first_id = first_id or videos[0]['id']
4353             last_id = videos[-1]['id']
4354             watch_endpoint = try_get(
4355                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4356             headers = self.generate_api_headers(
4357                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4358                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4359             query = {
4360                 'playlistId': playlist_id,
4361                 'videoId': watch_endpoint.get('videoId') or last_id,
4362                 'index': watch_endpoint.get('index') or len(videos),
4363                 'params': watch_endpoint.get('params') or 'OAE%3D'
4364             }
4365             response = self._extract_response(
4366                 item_id='%s page %d' % (playlist_id, page_num),
4367                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4368                 check_get_keys='contents'
4369             )
4370             playlist = try_get(
4371                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4372
4373     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4374         title = playlist.get('title') or try_get(
4375             data, lambda x: x['titleText']['simpleText'], str)
4376         playlist_id = playlist.get('playlistId') or item_id
4377
4378         # Delegating everything except mix playlists to regular tab-based playlist URL
4379         playlist_url = urljoin(url, try_get(
4380             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4381             str))
4382
4383         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4384         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4385         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4386
4387         if playlist_url and playlist_url != url and not is_known_unviewable:
4388             return self.url_result(
4389                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4390                 video_title=title)
4391
4392         return self.playlist_result(
4393             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4394             playlist_id=playlist_id, playlist_title=title)
4395
4396     def _extract_availability(self, data):
4397         """
4398         Gets the availability of a given playlist/tab.
4399         Note: Unless YouTube tells us explicitly, we do not assume it is public
4400         @param data: response
4401         """
4402         is_private = is_unlisted = None
4403         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4404         badge_labels = self._extract_badges(renderer)
4405
4406         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4407         privacy_dropdown_entries = try_get(
4408             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4409         for renderer_dict in privacy_dropdown_entries:
4410             is_selected = try_get(
4411                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4412             if not is_selected:
4413                 continue
4414             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4415             if label:
4416                 badge_labels.add(label.lower())
4417                 break
4418
4419         for badge_label in badge_labels:
4420             if badge_label == 'unlisted':
4421                 is_unlisted = True
4422             elif badge_label == 'private':
4423                 is_private = True
4424             elif badge_label == 'public':
4425                 is_unlisted = is_private = False
4426         return self._availability(is_private, False, False, False, is_unlisted)
4427
4428     @staticmethod
4429     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4430         sidebar_renderer = try_get(
4431             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4432         for item in sidebar_renderer:
4433             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4434             if renderer:
4435                 return renderer
4436
4437     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4438         """
4439         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4440         """
4441         browse_id = params = None
4442         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4443         if not renderer:
4444             return
4445         menu_renderer = try_get(
4446             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4447         for menu_item in menu_renderer:
4448             if not isinstance(menu_item, dict):
4449                 continue
4450             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4451             text = try_get(
4452                 nav_item_renderer, lambda x: x['text']['simpleText'], str)
4453             if not text or text.lower() != 'show unavailable videos':
4454                 continue
4455             browse_endpoint = try_get(
4456                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4457             browse_id = browse_endpoint.get('browseId')
4458             params = browse_endpoint.get('params')
4459             break
4460
4461         headers = self.generate_api_headers(
4462             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4463             visitor_data=self._extract_visitor_data(data, ytcfg))
4464         query = {
4465             'params': params or 'wgYCCAA=',
4466             'browseId': browse_id or 'VL%s' % item_id
4467         }
4468         return self._extract_response(
4469             item_id=item_id, headers=headers, query=query,
4470             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4471             note='Downloading API JSON with unavailable videos')
4472
4473     @functools.cached_property
4474     def skip_webpage(self):
4475         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4476
4477     def _extract_webpage(self, url, item_id, fatal=True):
4478         retries = self.get_param('extractor_retries', 3)
4479         count = -1
4480         webpage = data = last_error = None
4481         while count < retries:
4482             count += 1
4483             # Sometimes youtube returns a webpage with incomplete ytInitialData
4484             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4485             if last_error:
4486                 self.report_warning('%s. Retrying ...' % last_error)
4487             try:
4488                 webpage = self._download_webpage(
4489                     url, item_id,
4490                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4491                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4492             except ExtractorError as e:
4493                 if isinstance(e.cause, network_exceptions):
4494                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4495                         last_error = error_to_compat_str(e.cause or e.msg)
4496                         if count < retries:
4497                             continue
4498                 if fatal:
4499                     raise
4500                 self.report_warning(error_to_compat_str(e))
4501                 break
4502             else:
4503                 try:
4504                     self._extract_and_report_alerts(data)
4505                 except ExtractorError as e:
4506                     if fatal:
4507                         raise
4508                     self.report_warning(error_to_compat_str(e))
4509                     break
4510
4511                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4512                     break
4513
4514                 last_error = 'Incomplete yt initial data received'
4515                 if count >= retries:
4516                     if fatal:
4517                         raise ExtractorError(last_error)
4518                     self.report_warning(last_error)
4519                     break
4520
4521         return webpage, data
4522
4523     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4524         """Use if failed to extract ytcfg (and data) from initial webpage"""
4525         if not ytcfg and self.is_authenticated:
4526             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4527             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4528                 raise ExtractorError(
4529                     f'{msg}. If you are not downloading private content, or '
4530                     'your cookies are only for the first account and channel,'
4531                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4532                     expected=True)
4533             self.report_warning(msg, only_once=True)
4534
4535     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4536         data = None
4537         if not self.skip_webpage:
4538             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4539             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4540             # Reject webpage data if redirected to home page without explicitly requesting
4541             selected_tab = self._extract_selected_tab(traverse_obj(
4542                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4543             if (url != 'https://www.youtube.com/feed/recommended'
4544                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4545                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4546                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4547                 if fatal:
4548                     raise ExtractorError(msg, expected=True)
4549                 self.report_warning(msg, only_once=True)
4550         if not data:
4551             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4552             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4553         return data, ytcfg
4554
4555     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4556         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4557         resolve_response = self._extract_response(
4558             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4559             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4560         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4561         for ep_key, ep in endpoints.items():
4562             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4563             if params:
4564                 return self._extract_response(
4565                     item_id=item_id, query=params, ep=ep, headers=headers,
4566                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4567                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4568         err_note = 'Failed to resolve url (does the playlist exist?)'
4569         if fatal:
4570             raise ExtractorError(err_note, expected=True)
4571         self.report_warning(err_note, item_id)
4572
4573     _SEARCH_PARAMS = None
4574
4575     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4576         data = {'query': query}
4577         if params is NO_DEFAULT:
4578             params = self._SEARCH_PARAMS
4579         if params:
4580             data['params'] = params
4581
4582         content_keys = (
4583             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4584             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4585             # ytmusic search
4586             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4587             ('continuationContents', ),
4588         )
4589         display_id = f'query "{query}"'
4590         check_get_keys = tuple({keys[0] for keys in content_keys})
4591         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4592         self._report_playlist_authcheck(ytcfg, fatal=False)
4593
4594         continuation_list = [None]
4595         search = None
4596         for page_num in itertools.count(1):
4597             data.update(continuation_list[0] or {})
4598             headers = self.generate_api_headers(
4599                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4600             search = self._extract_response(
4601                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4602                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4603             slr_contents = traverse_obj(search, *content_keys)
4604             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4605             if not continuation_list[0]:
4606                 break
4607
4608
4609 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4610     IE_DESC = 'YouTube Tabs'
4611     _VALID_URL = r'''(?x:
4612         https?://
4613             (?:\w+\.)?
4614             (?:
4615                 youtube(?:kids)?\.com|
4616                 %(invidious)s
4617             )/
4618             (?:
4619                 (?P<channel_type>channel|c|user|browse)/|
4620                 (?P<not_channel>
4621                     feed/|hashtag/|
4622                     (?:playlist|watch)\?.*?\blist=
4623                 )|
4624                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4625             )
4626             (?P<id>[^/?\#&]+)
4627     )''' % {
4628         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4629         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4630     }
4631     IE_NAME = 'youtube:tab'
4632
4633     _TESTS = [{
4634         'note': 'playlists, multipage',
4635         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4636         'playlist_mincount': 94,
4637         'info_dict': {
4638             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4639             'title': 'Igor Kleiner - Playlists',
4640             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4641             'uploader': 'Igor Kleiner',
4642             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4643             'channel': 'Igor Kleiner',
4644             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4645             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4646             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4647             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4648             'channel_follower_count': int
4649         },
4650     }, {
4651         'note': 'playlists, multipage, different order',
4652         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4653         'playlist_mincount': 94,
4654         'info_dict': {
4655             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4656             'title': 'Igor Kleiner - Playlists',
4657             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4658             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4659             'uploader': 'Igor Kleiner',
4660             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4661             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4662             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4663             'channel': 'Igor Kleiner',
4664             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4665             'channel_follower_count': int
4666         },
4667     }, {
4668         'note': 'playlists, series',
4669         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4670         'playlist_mincount': 5,
4671         'info_dict': {
4672             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4673             'title': '3Blue1Brown - Playlists',
4674             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4675             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4676             'uploader': '3Blue1Brown',
4677             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4678             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4679             'channel': '3Blue1Brown',
4680             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4681             'tags': ['Mathematics'],
4682             'channel_follower_count': int
4683         },
4684     }, {
4685         'note': 'playlists, singlepage',
4686         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4687         'playlist_mincount': 4,
4688         'info_dict': {
4689             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4690             'title': 'ThirstForScience - Playlists',
4691             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4692             'uploader': 'ThirstForScience',
4693             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4694             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4695             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4696             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4697             'tags': 'count:13',
4698             'channel': 'ThirstForScience',
4699             'channel_follower_count': int
4700         }
4701     }, {
4702         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4703         'only_matching': True,
4704     }, {
4705         'note': 'basic, single video playlist',
4706         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4707         'info_dict': {
4708             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4709             'uploader': 'Sergey M.',
4710             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4711             'title': 'youtube-dl public playlist',
4712             'description': '',
4713             'tags': [],
4714             'view_count': int,
4715             'modified_date': '20201130',
4716             'channel': 'Sergey M.',
4717             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4718             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4719             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4720         },
4721         'playlist_count': 1,
4722     }, {
4723         'note': 'empty playlist',
4724         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4725         'info_dict': {
4726             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4727             'uploader': 'Sergey M.',
4728             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4729             'title': 'youtube-dl empty playlist',
4730             'tags': [],
4731             'channel': 'Sergey M.',
4732             'description': '',
4733             'modified_date': '20160902',
4734             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4735             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4736             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4737         },
4738         'playlist_count': 0,
4739     }, {
4740         'note': 'Home tab',
4741         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4742         'info_dict': {
4743             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4744             'title': 'lex will - Home',
4745             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4746             'uploader': 'lex will',
4747             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4748             'channel': 'lex will',
4749             'tags': ['bible', 'history', 'prophesy'],
4750             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4751             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4752             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4753             'channel_follower_count': int
4754         },
4755         'playlist_mincount': 2,
4756     }, {
4757         'note': 'Videos tab',
4758         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4759         'info_dict': {
4760             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4761             'title': 'lex will - Videos',
4762             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4763             'uploader': 'lex will',
4764             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4765             'tags': ['bible', 'history', 'prophesy'],
4766             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4767             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4768             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4769             'channel': 'lex will',
4770             'channel_follower_count': int
4771         },
4772         'playlist_mincount': 975,
4773     }, {
4774         'note': 'Videos tab, sorted by popular',
4775         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4776         'info_dict': {
4777             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4778             'title': 'lex will - Videos',
4779             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4780             'uploader': 'lex will',
4781             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4782             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4783             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4784             'channel': 'lex will',
4785             'tags': ['bible', 'history', 'prophesy'],
4786             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4787             'channel_follower_count': int
4788         },
4789         'playlist_mincount': 199,
4790     }, {
4791         'note': 'Playlists tab',
4792         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4793         'info_dict': {
4794             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4795             'title': 'lex will - Playlists',
4796             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4797             'uploader': 'lex will',
4798             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4799             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4800             'channel': 'lex will',
4801             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4802             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4803             'tags': ['bible', 'history', 'prophesy'],
4804             'channel_follower_count': int
4805         },
4806         'playlist_mincount': 17,
4807     }, {
4808         'note': 'Community tab',
4809         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4810         'info_dict': {
4811             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4812             'title': 'lex will - Community',
4813             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4814             'uploader': 'lex will',
4815             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4816             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4817             'channel': 'lex will',
4818             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4819             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4820             'tags': ['bible', 'history', 'prophesy'],
4821             'channel_follower_count': int
4822         },
4823         'playlist_mincount': 18,
4824     }, {
4825         'note': 'Channels tab',
4826         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4827         'info_dict': {
4828             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4829             'title': 'lex will - Channels',
4830             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4831             'uploader': 'lex will',
4832             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4833             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4834             'channel': 'lex will',
4835             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4836             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4837             'tags': ['bible', 'history', 'prophesy'],
4838             'channel_follower_count': int
4839         },
4840         'playlist_mincount': 12,
4841     }, {
4842         'note': 'Search tab',
4843         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4844         'playlist_mincount': 40,
4845         'info_dict': {
4846             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4847             'title': '3Blue1Brown - Search - linear algebra',
4848             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4849             'uploader': '3Blue1Brown',
4850             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4851             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4852             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4853             'tags': ['Mathematics'],
4854             'channel': '3Blue1Brown',
4855             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4856             'channel_follower_count': int
4857         },
4858     }, {
4859         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4860         'only_matching': True,
4861     }, {
4862         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4863         'only_matching': True,
4864     }, {
4865         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4866         'only_matching': True,
4867     }, {
4868         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4869         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4870         'info_dict': {
4871             'title': '29C3: Not my department',
4872             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4873             'uploader': 'Christiaan008',
4874             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4875             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4876             'tags': [],
4877             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4878             'view_count': int,
4879             'modified_date': '20150605',
4880             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4881             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4882             'channel': 'Christiaan008',
4883         },
4884         'playlist_count': 96,
4885     }, {
4886         'note': 'Large playlist',
4887         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4888         'info_dict': {
4889             'title': 'Uploads from Cauchemar',
4890             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4891             'uploader': 'Cauchemar',
4892             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4893             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4894             'tags': [],
4895             'modified_date': r're:\d{8}',
4896             'channel': 'Cauchemar',
4897             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4898             'view_count': int,
4899             'description': '',
4900             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4901         },
4902         'playlist_mincount': 1123,
4903         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4904     }, {
4905         'note': 'even larger playlist, 8832 videos',
4906         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4907         'only_matching': True,
4908     }, {
4909         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4910         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4911         'info_dict': {
4912             'title': 'Uploads from Interstellar Movie',
4913             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4914             'uploader': 'Interstellar Movie',
4915             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4916             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4917             'tags': [],
4918             'view_count': int,
4919             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4920             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4921             'channel': 'Interstellar Movie',
4922             'description': '',
4923             'modified_date': r're:\d{8}',
4924         },
4925         'playlist_mincount': 21,
4926     }, {
4927         'note': 'Playlist with "show unavailable videos" button',
4928         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4929         'info_dict': {
4930             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4931             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4932             'uploader': 'Phim Siêu Nhân Nhật Bản',
4933             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4934             'view_count': int,
4935             'channel': 'Phim Siêu Nhân Nhật Bản',
4936             'tags': [],
4937             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4938             'description': '',
4939             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4940             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4941             'modified_date': r're:\d{8}',
4942         },
4943         'playlist_mincount': 200,
4944         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4945     }, {
4946         'note': 'Playlist with unavailable videos in page 7',
4947         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4948         'info_dict': {
4949             'title': 'Uploads from BlankTV',
4950             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4951             'uploader': 'BlankTV',
4952             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4953             'channel': 'BlankTV',
4954             'channel_url': 'https://www.youtube.com/c/blanktv',
4955             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4956             'view_count': int,
4957             'tags': [],
4958             'uploader_url': 'https://www.youtube.com/c/blanktv',
4959             'modified_date': r're:\d{8}',
4960             'description': '',
4961         },
4962         'playlist_mincount': 1000,
4963         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4964     }, {
4965         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4966         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4967         'info_dict': {
4968             'title': 'Data Analysis with Dr Mike Pound',
4969             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4970             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4971             'uploader': 'Computerphile',
4972             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4973             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4974             'tags': [],
4975             'view_count': int,
4976             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4977             'channel_url': 'https://www.youtube.com/user/Computerphile',
4978             'channel': 'Computerphile',
4979         },
4980         'playlist_mincount': 11,
4981     }, {
4982         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4983         'only_matching': True,
4984     }, {
4985         'note': 'Playlist URL that does not actually serve a playlist',
4986         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4987         'info_dict': {
4988             'id': 'FqZTN594JQw',
4989             'ext': 'webm',
4990             'title': "Smiley's People 01 detective, Adventure Series, Action",
4991             'uploader': 'STREEM',
4992             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4993             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4994             'upload_date': '20150526',
4995             'license': 'Standard YouTube License',
4996             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4997             'categories': ['People & Blogs'],
4998             'tags': list,
4999             'view_count': int,
5000             'like_count': int,
5001         },
5002         'params': {
5003             'skip_download': True,
5004         },
5005         'skip': 'This video is not available.',
5006         'add_ie': [YoutubeIE.ie_key()],
5007     }, {
5008         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5009         'only_matching': True,
5010     }, {
5011         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5012         'only_matching': True,
5013     }, {
5014         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5015         'info_dict': {
5016             'id': 'Wq15eF5vCbI',  # This will keep changing
5017             'ext': 'mp4',
5018             'title': str,
5019             'uploader': 'Sky News',
5020             'uploader_id': 'skynews',
5021             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5022             'upload_date': r're:\d{8}',
5023             'description': str,
5024             'categories': ['News & Politics'],
5025             'tags': list,
5026             'like_count': int,
5027             'release_timestamp': 1642502819,
5028             'channel': 'Sky News',
5029             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5030             'age_limit': 0,
5031             'view_count': int,
5032             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5033             'playable_in_embed': True,
5034             'release_date': '20220118',
5035             'availability': 'public',
5036             'live_status': 'is_live',
5037             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5038             'channel_follower_count': int
5039         },
5040         'params': {
5041             'skip_download': True,
5042         },
5043         'expected_warnings': ['Ignoring subtitle tracks found in '],
5044     }, {
5045         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5046         'info_dict': {
5047             'id': 'a48o2S1cPoo',
5048             'ext': 'mp4',
5049             'title': 'The Young Turks - Live Main Show',
5050             'uploader': 'The Young Turks',
5051             'uploader_id': 'TheYoungTurks',
5052             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5053             'upload_date': '20150715',
5054             'license': 'Standard YouTube License',
5055             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5056             'categories': ['News & Politics'],
5057             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5058             'like_count': int,
5059         },
5060         'params': {
5061             'skip_download': True,
5062         },
5063         'only_matching': True,
5064     }, {
5065         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5066         'only_matching': True,
5067     }, {
5068         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5069         'only_matching': True,
5070     }, {
5071         'note': 'A channel that is not live. Should raise error',
5072         'url': 'https://www.youtube.com/user/numberphile/live',
5073         'only_matching': True,
5074     }, {
5075         'url': 'https://www.youtube.com/feed/trending',
5076         'only_matching': True,
5077     }, {
5078         'url': 'https://www.youtube.com/feed/library',
5079         'only_matching': True,
5080     }, {
5081         'url': 'https://www.youtube.com/feed/history',
5082         'only_matching': True,
5083     }, {
5084         'url': 'https://www.youtube.com/feed/subscriptions',
5085         'only_matching': True,
5086     }, {
5087         'url': 'https://www.youtube.com/feed/watch_later',
5088         'only_matching': True,
5089     }, {
5090         'note': 'Recommended - redirects to home page.',
5091         'url': 'https://www.youtube.com/feed/recommended',
5092         'only_matching': True,
5093     }, {
5094         'note': 'inline playlist with not always working continuations',
5095         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5096         'only_matching': True,
5097     }, {
5098         'url': 'https://www.youtube.com/course',
5099         'only_matching': True,
5100     }, {
5101         'url': 'https://www.youtube.com/zsecurity',
5102         'only_matching': True,
5103     }, {
5104         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5105         'only_matching': True,
5106     }, {
5107         'url': 'https://www.youtube.com/TheYoungTurks/live',
5108         'only_matching': True,
5109     }, {
5110         'url': 'https://www.youtube.com/hashtag/cctv9',
5111         'info_dict': {
5112             'id': 'cctv9',
5113             'title': '#cctv9',
5114             'tags': [],
5115         },
5116         'playlist_mincount': 350,
5117     }, {
5118         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5119         'only_matching': True,
5120     }, {
5121         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5122         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5123         'only_matching': True
5124     }, {
5125         'note': '/browse/ should redirect to /channel/',
5126         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5127         'only_matching': True
5128     }, {
5129         'note': 'VLPL, should redirect to playlist?list=PL...',
5130         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5131         'info_dict': {
5132             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5133             'uploader': 'NoCopyrightSounds',
5134             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5135             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5136             'title': 'NCS : All Releases 💿',
5137             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5138             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5139             'modified_date': r're:\d{8}',
5140             'view_count': int,
5141             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5142             'tags': [],
5143             'channel': 'NoCopyrightSounds',
5144         },
5145         'playlist_mincount': 166,
5146         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5147     }, {
5148         'note': 'Topic, should redirect to playlist?list=UU...',
5149         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5150         'info_dict': {
5151             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5152             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5153             'title': 'Uploads from Royalty Free Music - Topic',
5154             'uploader': 'Royalty Free Music - Topic',
5155             'tags': [],
5156             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5157             'channel': 'Royalty Free Music - Topic',
5158             'view_count': int,
5159             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5160             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5161             'modified_date': r're:\d{8}',
5162             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5163             'description': '',
5164         },
5165         'expected_warnings': [
5166             'The URL does not have a videos tab',
5167             r'[Uu]navailable videos (are|will be) hidden',
5168         ],
5169         'playlist_mincount': 101,
5170     }, {
5171         'note': 'Topic without a UU playlist',
5172         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5173         'info_dict': {
5174             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5175             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5176             'tags': [],
5177         },
5178         'expected_warnings': [
5179             'the playlist redirect gave error',
5180         ],
5181         'playlist_mincount': 9,
5182     }, {
5183         'note': 'Youtube music Album',
5184         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5185         'info_dict': {
5186             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5187             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5188             'tags': [],
5189             'view_count': int,
5190             'description': '',
5191             'availability': 'unlisted',
5192             'modified_date': r're:\d{8}',
5193         },
5194         'playlist_count': 50,
5195     }, {
5196         'note': 'unlisted single video playlist',
5197         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5198         'info_dict': {
5199             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5200             'uploader': 'colethedj',
5201             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5202             'title': 'yt-dlp unlisted playlist test',
5203             'availability': 'unlisted',
5204             'tags': [],
5205             'modified_date': '20220418',
5206             'channel': 'colethedj',
5207             'view_count': int,
5208             'description': '',
5209             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5210             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5211             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5212         },
5213         'playlist_count': 1,
5214     }, {
5215         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5216         'url': 'https://www.youtube.com/feed/recommended',
5217         'info_dict': {
5218             'id': 'recommended',
5219             'title': 'recommended',
5220             'tags': [],
5221         },
5222         'playlist_mincount': 50,
5223         'params': {
5224             'skip_download': True,
5225             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5226         },
5227     }, {
5228         'note': 'API Fallback: /videos tab, sorted by oldest first',
5229         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5230         'info_dict': {
5231             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5232             'title': 'Cody\'sLab - Videos',
5233             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5234             'uploader': 'Cody\'sLab',
5235             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5236             'channel': 'Cody\'sLab',
5237             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5238             'tags': [],
5239             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5240             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5241             'channel_follower_count': int
5242         },
5243         'playlist_mincount': 650,
5244         'params': {
5245             'skip_download': True,
5246             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5247         },
5248     }, {
5249         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5250         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5251         'info_dict': {
5252             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5253             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5254             'title': 'Uploads from Royalty Free Music - Topic',
5255             'uploader': 'Royalty Free Music - Topic',
5256             'modified_date': r're:\d{8}',
5257             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5258             'description': '',
5259             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5260             'tags': [],
5261             'channel': 'Royalty Free Music - Topic',
5262             'view_count': int,
5263             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5264         },
5265         'expected_warnings': [
5266             'does not have a videos tab',
5267             r'[Uu]navailable videos (are|will be) hidden',
5268         ],
5269         'playlist_mincount': 101,
5270         'params': {
5271             'skip_download': True,
5272             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5273         },
5274     }, {
5275         'note': 'non-standard redirect to regional channel',
5276         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5277         'only_matching': True
5278     }, {
5279         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5280         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5281         'info_dict': {
5282             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5283             'modified_date': '20220407',
5284             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5285             'tags': [],
5286             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5287             'uploader': 'pukkandan',
5288             'availability': 'unlisted',
5289             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5290             'channel': 'pukkandan',
5291             'description': 'Test for collaborative playlist',
5292             'title': 'yt-dlp test - collaborative playlist',
5293             'view_count': int,
5294             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5295         },
5296         'playlist_mincount': 2
5297     }]
5298
5299     @classmethod
5300     def suitable(cls, url):
5301         return False if YoutubeIE.suitable(url) else super().suitable(url)
5302
5303     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5304
5305     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5306     def _real_extract(self, url, smuggled_data):
5307         item_id = self._match_id(url)
5308         url = urllib.parse.urlunparse(
5309             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5310         compat_opts = self.get_param('compat_opts', [])
5311
5312         def get_mobj(url):
5313             mobj = self._URL_RE.match(url).groupdict()
5314             mobj.update((k, '') for k, v in mobj.items() if v is None)
5315             return mobj
5316
5317         mobj, redirect_warning = get_mobj(url), None
5318         # Youtube returns incomplete data if tabname is not lower case
5319         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5320         if is_channel:
5321             if smuggled_data.get('is_music_url'):
5322                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5323                     item_id = item_id[2:]
5324                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5325                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5326                     mdata = self._extract_tab_endpoint(
5327                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5328                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5329                                         get_all=False, expected_type=str)
5330                     if not murl:
5331                         raise ExtractorError('Failed to resolve album to playlist')
5332                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5333                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5334                     pre = f'https://www.youtube.com/channel/{item_id}'
5335
5336         original_tab_name = tab
5337         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5338             # Home URLs should redirect to /videos/
5339             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5340                                 'To download only the videos in the home page, add a "/featured" to the URL')
5341             tab = '/videos'
5342
5343         url = ''.join((pre, tab, post))
5344         mobj = get_mobj(url)
5345
5346         # Handle both video/playlist URLs
5347         qs = parse_qs(url)
5348         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5349
5350         if not video_id and mobj['not_channel'].startswith('watch'):
5351             if not playlist_id:
5352                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5353                 raise ExtractorError('Unable to recognize tab page')
5354             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5355             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5356             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5357             mobj = get_mobj(url)
5358
5359         if video_id and playlist_id:
5360             if self.get_param('noplaylist'):
5361                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5362                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5363                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5364             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5365
5366         data, ytcfg = self._extract_data(url, item_id)
5367
5368         # YouTube may provide a non-standard redirect to the regional channel
5369         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5370         redirect_url = traverse_obj(
5371             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5372         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5373             redirect_url = ''.join((
5374                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5375             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5376             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5377
5378         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5379         if tabs:
5380             selected_tab = self._extract_selected_tab(tabs)
5381             selected_tab_name = selected_tab.get('title', '').lower()
5382             if selected_tab_name == 'home':
5383                 selected_tab_name = 'featured'
5384             requested_tab_name = mobj['tab'][1:]
5385             if 'no-youtube-channel-redirect' not in compat_opts:
5386                 if requested_tab_name == 'live':
5387                     # Live tab should have redirected to the video
5388                     raise ExtractorError('The channel is not currently live', expected=True)
5389                 if requested_tab_name not in ('', selected_tab_name):
5390                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5391                     if not original_tab_name:
5392                         if item_id[:2] == 'UC':
5393                             # Topic channels don't have /videos. Use the equivalent playlist instead
5394                             pl_id = f'UU{item_id[2:]}'
5395                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5396                             try:
5397                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5398                             except ExtractorError:
5399                                 redirect_warning += ' and the playlist redirect gave error'
5400                             else:
5401                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5402                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5403                         if selected_tab_name and selected_tab_name != requested_tab_name:
5404                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5405                     else:
5406                         raise ExtractorError(redirect_warning, expected=True)
5407
5408         if redirect_warning:
5409             self.to_screen(redirect_warning)
5410         self.write_debug(f'Final URL: {url}')
5411
5412         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5413         if 'no-youtube-unavailable-videos' not in compat_opts:
5414             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5415         self._extract_and_report_alerts(data, only_once=True)
5416         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5417         if tabs:
5418             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5419
5420         playlist = traverse_obj(
5421             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5422         if playlist:
5423             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5424
5425         video_id = traverse_obj(
5426             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5427         if video_id:
5428             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5429                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5430             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5431                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5432
5433         raise ExtractorError('Unable to recognize tab page')
5434
5435
5436 class YoutubePlaylistIE(InfoExtractor):
5437     IE_DESC = 'YouTube playlists'
5438     _VALID_URL = r'''(?x)(?:
5439                         (?:https?://)?
5440                         (?:\w+\.)?
5441                         (?:
5442                             (?:
5443                                 youtube(?:kids)?\.com|
5444                                 %(invidious)s
5445                             )
5446                             /.*?\?.*?\blist=
5447                         )?
5448                         (?P<id>%(playlist_id)s)
5449                      )''' % {
5450         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5451         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5452     }
5453     IE_NAME = 'youtube:playlist'
5454     _TESTS = [{
5455         'note': 'issue #673',
5456         'url': 'PLBB231211A4F62143',
5457         'info_dict': {
5458             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5459             'id': 'PLBB231211A4F62143',
5460             'uploader': 'Wickman',
5461             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5462             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5463             'view_count': int,
5464             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5465             'modified_date': r're:\d{8}',
5466             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5467             'channel': 'Wickman',
5468             'tags': [],
5469             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5470         },
5471         'playlist_mincount': 29,
5472     }, {
5473         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5474         'info_dict': {
5475             'title': 'YDL_safe_search',
5476             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5477         },
5478         'playlist_count': 2,
5479         'skip': 'This playlist is private',
5480     }, {
5481         'note': 'embedded',
5482         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5483         'playlist_count': 4,
5484         'info_dict': {
5485             'title': 'JODA15',
5486             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5487             'uploader': 'milan',
5488             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5489             'description': '',
5490             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5491             'tags': [],
5492             'modified_date': '20140919',
5493             'view_count': int,
5494             'channel': 'milan',
5495             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5496             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5497         },
5498         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5499     }, {
5500         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5501         'playlist_mincount': 455,
5502         'info_dict': {
5503             'title': '2018 Chinese New Singles (11/6 updated)',
5504             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5505             'uploader': 'LBK',
5506             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5507             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5508             'channel': 'LBK',
5509             'view_count': int,
5510             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5511             'tags': [],
5512             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5513             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5514             'modified_date': r're:\d{8}',
5515         },
5516         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5517     }, {
5518         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5519         'only_matching': True,
5520     }, {
5521         # music album playlist
5522         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5523         'only_matching': True,
5524     }]
5525
5526     @classmethod
5527     def suitable(cls, url):
5528         if YoutubeTabIE.suitable(url):
5529             return False
5530         from ..utils import parse_qs
5531         qs = parse_qs(url)
5532         if qs.get('v', [None])[0]:
5533             return False
5534         return super().suitable(url)
5535
5536     def _real_extract(self, url):
5537         playlist_id = self._match_id(url)
5538         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5539         url = update_url_query(
5540             'https://www.youtube.com/playlist',
5541             parse_qs(url) or {'list': playlist_id})
5542         if is_music_url:
5543             url = smuggle_url(url, {'is_music_url': True})
5544         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5545
5546
5547 class YoutubeYtBeIE(InfoExtractor):
5548     IE_DESC = 'youtu.be'
5549     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5550     _TESTS = [{
5551         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5552         'info_dict': {
5553             'id': 'yeWKywCrFtk',
5554             'ext': 'mp4',
5555             'title': 'Small Scale Baler and Braiding Rugs',
5556             'uploader': 'Backus-Page House Museum',
5557             'uploader_id': 'backuspagemuseum',
5558             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5559             'upload_date': '20161008',
5560             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5561             'categories': ['Nonprofits & Activism'],
5562             'tags': list,
5563             'like_count': int,
5564             'age_limit': 0,
5565             'playable_in_embed': True,
5566             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5567             'channel': 'Backus-Page House Museum',
5568             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5569             'live_status': 'not_live',
5570             'view_count': int,
5571             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5572             'availability': 'public',
5573             'duration': 59,
5574             'comment_count': int,
5575             'channel_follower_count': int
5576         },
5577         'params': {
5578             'noplaylist': True,
5579             'skip_download': True,
5580         },
5581     }, {
5582         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5583         'only_matching': True,
5584     }]
5585
5586     def _real_extract(self, url):
5587         mobj = self._match_valid_url(url)
5588         video_id = mobj.group('id')
5589         playlist_id = mobj.group('playlist_id')
5590         return self.url_result(
5591             update_url_query('https://www.youtube.com/watch', {
5592                 'v': video_id,
5593                 'list': playlist_id,
5594                 'feature': 'youtu.be',
5595             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5596
5597
5598 class YoutubeLivestreamEmbedIE(InfoExtractor):
5599     IE_DESC = 'YouTube livestream embeds'
5600     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5601     _TESTS = [{
5602         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5603         'only_matching': True,
5604     }]
5605
5606     def _real_extract(self, url):
5607         channel_id = self._match_id(url)
5608         return self.url_result(
5609             f'https://www.youtube.com/channel/{channel_id}/live',
5610             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5611
5612
5613 class YoutubeYtUserIE(InfoExtractor):
5614     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5615     IE_NAME = 'youtube:user'
5616     _VALID_URL = r'ytuser:(?P<id>.+)'
5617     _TESTS = [{
5618         'url': 'ytuser:phihag',
5619         'only_matching': True,
5620     }]
5621
5622     def _real_extract(self, url):
5623         user_id = self._match_id(url)
5624         return self.url_result(
5625             'https://www.youtube.com/user/%s/videos' % user_id,
5626             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5627
5628
5629 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5630     IE_NAME = 'youtube:favorites'
5631     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5632     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5633     _LOGIN_REQUIRED = True
5634     _TESTS = [{
5635         'url': ':ytfav',
5636         'only_matching': True,
5637     }, {
5638         'url': ':ytfavorites',
5639         'only_matching': True,
5640     }]
5641
5642     def _real_extract(self, url):
5643         return self.url_result(
5644             'https://www.youtube.com/playlist?list=LL',
5645             ie=YoutubeTabIE.ie_key())
5646
5647
5648 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5649     IE_NAME = 'youtube:notif'
5650     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5651     _VALID_URL = r':ytnotif(?:ication)?s?'
5652     _LOGIN_REQUIRED = True
5653     _TESTS = [{
5654         'url': ':ytnotif',
5655         'only_matching': True,
5656     }, {
5657         'url': ':ytnotifications',
5658         'only_matching': True,
5659     }]
5660
5661     def _extract_notification_menu(self, response, continuation_list):
5662         notification_list = traverse_obj(
5663             response,
5664             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5665             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5666             expected_type=list) or []
5667         continuation_list[0] = None
5668         for item in notification_list:
5669             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5670             if entry:
5671                 yield entry
5672             continuation = item.get('continuationItemRenderer')
5673             if continuation:
5674                 continuation_list[0] = continuation
5675
5676     def _extract_notification_renderer(self, notification):
5677         video_id = traverse_obj(
5678             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5679         url = f'https://www.youtube.com/watch?v={video_id}'
5680         channel_id = None
5681         if not video_id:
5682             browse_ep = traverse_obj(
5683                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5684             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5685             post_id = self._search_regex(
5686                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5687                 'post id', default=None)
5688             if not channel_id or not post_id:
5689                 return
5690             # The direct /post url redirects to this in the browser
5691             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5692
5693         channel = traverse_obj(
5694             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5695             expected_type=str)
5696         notification_title = self._get_text(notification, 'shortMessage')
5697         if notification_title:
5698             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
5699         # TODO: handle recommended videos
5700         title = self._search_regex(
5701             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
5702             'video title', default=None)
5703         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5704                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5705                        else None)
5706         return {
5707             '_type': 'url',
5708             'url': url,
5709             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5710             'video_id': video_id,
5711             'title': title,
5712             'channel_id': channel_id,
5713             'channel': channel,
5714             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5715             'upload_date': upload_date,
5716         }
5717
5718     def _notification_menu_entries(self, ytcfg):
5719         continuation_list = [None]
5720         response = None
5721         for page in itertools.count(1):
5722             ctoken = traverse_obj(
5723                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5724             response = self._extract_response(
5725                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5726                 ep='notification/get_notification_menu', check_get_keys='actions',
5727                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5728             yield from self._extract_notification_menu(response, continuation_list)
5729             if not continuation_list[0]:
5730                 break
5731
5732     def _real_extract(self, url):
5733         display_id = 'notifications'
5734         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5735         self._report_playlist_authcheck(ytcfg)
5736         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5737
5738
5739 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5740     IE_DESC = 'YouTube search'
5741     IE_NAME = 'youtube:search'
5742     _SEARCH_KEY = 'ytsearch'
5743     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5744     _TESTS = [{
5745         'url': 'ytsearch5:youtube-dl test video',
5746         'playlist_count': 5,
5747         'info_dict': {
5748             'id': 'youtube-dl test video',
5749             'title': 'youtube-dl test video',
5750         }
5751     }]
5752
5753
5754 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5755     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5756     _SEARCH_KEY = 'ytsearchdate'
5757     IE_DESC = 'YouTube search, newest videos first'
5758     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5759     _TESTS = [{
5760         'url': 'ytsearchdate5:youtube-dl test video',
5761         'playlist_count': 5,
5762         'info_dict': {
5763             'id': 'youtube-dl test video',
5764             'title': 'youtube-dl test video',
5765         }
5766     }]
5767
5768
5769 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5770     IE_DESC = 'YouTube search URLs with sorting and filter support'
5771     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5772     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5773     _TESTS = [{
5774         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5775         'playlist_mincount': 5,
5776         'info_dict': {
5777             'id': 'youtube-dl test video',
5778             'title': 'youtube-dl test video',
5779         }
5780     }, {
5781         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5782         'playlist_mincount': 5,
5783         'info_dict': {
5784             'id': 'python',
5785             'title': 'python',
5786         }
5787     }, {
5788         'url': 'https://www.youtube.com/results?search_query=%23cats',
5789         'playlist_mincount': 1,
5790         'info_dict': {
5791             'id': '#cats',
5792             'title': '#cats',
5793             # The test suite does not have support for nested playlists
5794             # 'entries': [{
5795             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5796             #     'title': '#cats',
5797             # }],
5798         },
5799     }, {
5800         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5801         'only_matching': True,
5802     }]
5803
5804     def _real_extract(self, url):
5805         qs = parse_qs(url)
5806         query = (qs.get('search_query') or qs.get('q'))[0]
5807         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5808
5809
5810 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5811     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5812     IE_NAME = 'youtube:music:search_url'
5813     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5814     _TESTS = [{
5815         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5816         'playlist_count': 16,
5817         'info_dict': {
5818             'id': 'royalty free music',
5819             'title': 'royalty free music',
5820         }
5821     }, {
5822         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5823         'playlist_mincount': 30,
5824         'info_dict': {
5825             'id': 'royalty free music - songs',
5826             'title': 'royalty free music - songs',
5827         },
5828         'params': {'extract_flat': 'in_playlist'}
5829     }, {
5830         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5831         'playlist_mincount': 30,
5832         'info_dict': {
5833             'id': 'royalty free music - community playlists',
5834             'title': 'royalty free music - community playlists',
5835         },
5836         'params': {'extract_flat': 'in_playlist'}
5837     }]
5838
5839     _SECTIONS = {
5840         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5841         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5842         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5843         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5844         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5845         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5846     }
5847
5848     def _real_extract(self, url):
5849         qs = parse_qs(url)
5850         query = (qs.get('search_query') or qs.get('q'))[0]
5851         params = qs.get('sp', (None,))[0]
5852         if params:
5853             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5854         else:
5855             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
5856             params = self._SECTIONS.get(section)
5857             if not params:
5858                 section = None
5859         title = join_nonempty(query, section, delim=' - ')
5860         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5861
5862
5863 class YoutubeFeedsInfoExtractor(InfoExtractor):
5864     """
5865     Base class for feed extractors
5866     Subclasses must re-define the _FEED_NAME property.
5867     """
5868     _LOGIN_REQUIRED = True
5869     _FEED_NAME = 'feeds'
5870
5871     def _real_initialize(self):
5872         YoutubeBaseInfoExtractor._check_login_required(self)
5873
5874     @classproperty
5875     def IE_NAME(self):
5876         return f'youtube:{self._FEED_NAME}'
5877
5878     def _real_extract(self, url):
5879         return self.url_result(
5880             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5881
5882
5883 class YoutubeWatchLaterIE(InfoExtractor):
5884     IE_NAME = 'youtube:watchlater'
5885     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5886     _VALID_URL = r':ytwatchlater'
5887     _TESTS = [{
5888         'url': ':ytwatchlater',
5889         'only_matching': True,
5890     }]
5891
5892     def _real_extract(self, url):
5893         return self.url_result(
5894             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5895
5896
5897 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5898     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5899     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5900     _FEED_NAME = 'recommended'
5901     _LOGIN_REQUIRED = False
5902     _TESTS = [{
5903         'url': ':ytrec',
5904         'only_matching': True,
5905     }, {
5906         'url': ':ytrecommended',
5907         'only_matching': True,
5908     }, {
5909         'url': 'https://youtube.com',
5910         'only_matching': True,
5911     }]
5912
5913
5914 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5915     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5916     _VALID_URL = r':ytsub(?:scription)?s?'
5917     _FEED_NAME = 'subscriptions'
5918     _TESTS = [{
5919         'url': ':ytsubs',
5920         'only_matching': True,
5921     }, {
5922         'url': ':ytsubscriptions',
5923         'only_matching': True,
5924     }]
5925
5926
5927 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5928     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5929     _VALID_URL = r':ythis(?:tory)?'
5930     _FEED_NAME = 'history'
5931     _TESTS = [{
5932         'url': ':ythistory',
5933         'only_matching': True,
5934     }]
5935
5936
5937 class YoutubeStoriesIE(InfoExtractor):
5938     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5939     IE_NAME = 'youtube:stories'
5940     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5941     _TESTS = [{
5942         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5943         'only_matching': True,
5944     }]
5945
5946     def _real_extract(self, url):
5947         playlist_id = f'RLTD{self._match_id(url)}'
5948         return self.url_result(
5949             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5950             ie=YoutubeTabIE, video_id=playlist_id)
5951
5952
5953 class YoutubeTruncatedURLIE(InfoExtractor):
5954     IE_NAME = 'youtube:truncated_url'
5955     IE_DESC = False  # Do not list
5956     _VALID_URL = r'''(?x)
5957         (?:https?://)?
5958         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5959         (?:watch\?(?:
5960             feature=[a-z_]+|
5961             annotation_id=annotation_[^&]+|
5962             x-yt-cl=[0-9]+|
5963             hl=[^&]*|
5964             t=[0-9]+
5965         )?
5966         |
5967             attribution_link\?a=[^&]+
5968         )
5969         $
5970     '''
5971
5972     _TESTS = [{
5973         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5974         'only_matching': True,
5975     }, {
5976         'url': 'https://www.youtube.com/watch?',
5977         'only_matching': True,
5978     }, {
5979         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5980         'only_matching': True,
5981     }, {
5982         'url': 'https://www.youtube.com/watch?feature=foo',
5983         'only_matching': True,
5984     }, {
5985         'url': 'https://www.youtube.com/watch?hl=en-GB',
5986         'only_matching': True,
5987     }, {
5988         'url': 'https://www.youtube.com/watch?t=2372',
5989         'only_matching': True,
5990     }]
5991
5992     def _real_extract(self, url):
5993         raise ExtractorError(
5994             'Did you forget to quote the URL? Remember that & is a meta '
5995             'character in most shells, so you want to put the URL in quotes, '
5996             'like  youtube-dl '
5997             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5998             ' or simply  youtube-dl BaW_jenozKc  .',
5999             expected=True)
6000
6001
6002 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6003     IE_NAME = 'youtube:clip'
6004     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6005     _TESTS = [{
6006         # FIXME: Other metadata should be extracted from the clip, not from the base video
6007         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6008         'info_dict': {
6009             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6010             'ext': 'mp4',
6011             'section_start': 29.0,
6012             'section_end': 39.7,
6013             'duration': 10.7,
6014             'age_limit': 0,
6015             'availability': 'public',
6016             'categories': ['Gaming'],
6017             'channel': 'Scott The Woz',
6018             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6019             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6020             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6021             'like_count': int,
6022             'playable_in_embed': True,
6023             'tags': 'count:17',
6024             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6025             'title': 'Mobile Games on Console - Scott The Woz',
6026             'upload_date': '20210920',
6027             'uploader': 'Scott The Woz',
6028             'uploader_id': 'scottthewoz',
6029             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6030             'view_count': int,
6031             'live_status': 'not_live',
6032             'channel_follower_count': int
6033         }
6034     }]
6035
6036     def _real_extract(self, url):
6037         clip_id = self._match_id(url)
6038         _, data = self._extract_webpage(url, clip_id)
6039
6040         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6041         if not video_id:
6042             raise ExtractorError('Unable to find video ID')
6043
6044         clip_data = traverse_obj(data, (
6045             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6046             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6047             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6048             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6049
6050         return {
6051             '_type': 'url_transparent',
6052             'url': f'https://www.youtube.com/watch?v={video_id}',
6053             'ie_key': YoutubeIE.ie_key(),
6054             'id': clip_id,
6055             'section_start': int(clip_data['startTimeMs']) / 1000,
6056             'section_end': int(clip_data['endTimeMs']) / 1000,
6057         }
6058
6059
6060 class YoutubeTruncatedIDIE(InfoExtractor):
6061     IE_NAME = 'youtube:truncated_id'
6062     IE_DESC = False  # Do not list
6063     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6064
6065     _TESTS = [{
6066         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6067         'only_matching': True,
6068     }]
6069
6070     def _real_extract(self, url):
6071         video_id = self._match_id(url)
6072         raise ExtractorError(
6073             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6074             expected=True)