yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import functools
   6 import hashlib
   7 import itertools
   8 import json
   9 import math
  10 import os.path
  11 import random
  12 import re
  13 import sys
  14 import threading
  15 import time
  16 import traceback
  17
  18 from .common import InfoExtractor, SearchInfoExtractor
  19 from ..compat import (
  20     compat_chr,
  21     compat_HTTPError,
  22     compat_parse_qs,
  23     compat_str,
  24     compat_urllib_parse_unquote_plus,
  25     compat_urllib_parse_urlencode,
  26     compat_urllib_parse_urlparse,
  27     compat_urlparse,
  28 )
  29 from ..jsinterp import JSInterpreter
  30 from ..utils import (
  31     NO_DEFAULT,
  32     ExtractorError,
  33     bug_reports_message,
  34     clean_html,
  35     datetime_from_str,
  36     dict_get,
  37     error_to_compat_str,
  38     float_or_none,
  39     format_field,
  40     get_first,
  41     int_or_none,
  42     is_html,
  43     join_nonempty,
  44     js_to_json,
  45     mimetype2ext,
  46     network_exceptions,
  47     orderedSet,
  48     parse_codecs,
  49     parse_count,
  50     parse_duration,
  51     parse_iso8601,
  52     parse_qs,
  53     qualities,
  54     remove_end,
  55     remove_start,
  56     smuggle_url,
  57     str_or_none,
  58     str_to_int,
  59     strftime_or_none,
  60     traverse_obj,
  61     try_get,
  62     unescapeHTML,
  63     unified_strdate,
  64     unified_timestamp,
  65     unsmuggle_url,
  66     update_url_query,
  67     url_or_none,
  68     urljoin,
  69     variadic,
  70 )
  71
  72 # any clients starting with _ cannot be explicity requested by the user
  73 INNERTUBE_CLIENTS = {
  74     'web': {
  75         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  76         'INNERTUBE_CONTEXT': {
  77             'client': {
  78                 'clientName': 'WEB',
  79                 'clientVersion': '2.20211221.00.00',
  80             }
  81         },
  82         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  83     },
  84     'web_embedded': {
  85         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  86         'INNERTUBE_CONTEXT': {
  87             'client': {
  88                 'clientName': 'WEB_EMBEDDED_PLAYER',
  89                 'clientVersion': '1.20211215.00.01',
  90             },
  91         },
  92         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  93     },
  94     'web_music': {
  95         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  96         'INNERTUBE_HOST': 'music.youtube.com',
  97         'INNERTUBE_CONTEXT': {
  98             'client': {
  99                 'clientName': 'WEB_REMIX',
 100                 'clientVersion': '1.20211213.00.00',
 101             }
 102         },
 103         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 104     },
 105     'web_creator': {
 106         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 107         'INNERTUBE_CONTEXT': {
 108             'client': {
 109                 'clientName': 'WEB_CREATOR',
 110                 'clientVersion': '1.20211220.02.00',
 111             }
 112         },
 113         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 114     },
 115     'android': {
 116         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 117         'INNERTUBE_CONTEXT': {
 118             'client': {
 119                 'clientName': 'ANDROID',
 120                 'clientVersion': '16.49',
 121             }
 122         },
 123         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 124         'REQUIRE_JS_PLAYER': False
 125     },
 126     'android_embedded': {
 127         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 128         'INNERTUBE_CONTEXT': {
 129             'client': {
 130                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 131                 'clientVersion': '16.49',
 132             },
 133         },
 134         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 135         'REQUIRE_JS_PLAYER': False
 136     },
 137     'android_music': {
 138         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 139         'INNERTUBE_CONTEXT': {
 140             'client': {
 141                 'clientName': 'ANDROID_MUSIC',
 142                 'clientVersion': '4.57',
 143             }
 144         },
 145         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 146         'REQUIRE_JS_PLAYER': False
 147     },
 148     'android_creator': {
 149         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 150         'INNERTUBE_CONTEXT': {
 151             'client': {
 152                 'clientName': 'ANDROID_CREATOR',
 153                 'clientVersion': '21.47',
 154             },
 155         },
 156         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 157         'REQUIRE_JS_PLAYER': False
 158     },
 159     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 160     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 161     'ios': {
 162         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 163         'INNERTUBE_CONTEXT': {
 164             'client': {
 165                 'clientName': 'IOS',
 166                 'clientVersion': '16.46',
 167                 'deviceModel': 'iPhone14,3',
 168             }
 169         },
 170         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 171         'REQUIRE_JS_PLAYER': False
 172     },
 173     'ios_embedded': {
 174         'INNERTUBE_CONTEXT': {
 175             'client': {
 176                 'clientName': 'IOS_MESSAGES_EXTENSION',
 177                 'clientVersion': '16.46',
 178                 'deviceModel': 'iPhone14,3',
 179             },
 180         },
 181         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 182         'REQUIRE_JS_PLAYER': False
 183     },
 184     'ios_music': {
 185         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 186         'INNERTUBE_CONTEXT': {
 187             'client': {
 188                 'clientName': 'IOS_MUSIC',
 189                 'clientVersion': '4.57',
 190             },
 191         },
 192         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 193         'REQUIRE_JS_PLAYER': False
 194     },
 195     'ios_creator': {
 196         'INNERTUBE_CONTEXT': {
 197             'client': {
 198                 'clientName': 'IOS_CREATOR',
 199                 'clientVersion': '21.47',
 200             },
 201         },
 202         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 203         'REQUIRE_JS_PLAYER': False
 204     },
 205     # mweb has 'ultralow' formats
 206     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 207     'mweb': {
 208         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 209         'INNERTUBE_CONTEXT': {
 210             'client': {
 211                 'clientName': 'MWEB',
 212                 'clientVersion': '2.20211221.01.00',
 213             }
 214         },
 215         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 216     },
 217     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 218     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 219     'tv_embedded': {
 220         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 221         'INNERTUBE_CONTEXT': {
 222             'client': {
 223                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 224                 'clientVersion': '2.0',
 225             },
 226         },
 227         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 228     },
 229 }
 230
 231
 232 def _split_innertube_client(client_name):
 233     variant, *base = client_name.rsplit('.', 1)
 234     if base:
 235         return variant, base[0], variant
 236     base, *variant = client_name.split('_', 1)
 237     return client_name, base, variant[0] if variant else None
 238
 239
 240 def build_innertube_clients():
 241     THIRD_PARTY = {
 242         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 243     }
 244     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 245     priority = qualities(BASE_CLIENTS[::-1])
 246
 247     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 248         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 249         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 250         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 251         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 252
 253         _, base_client, variant = _split_innertube_client(client)
 254         ytcfg['priority'] = 10 * priority(base_client)
 255
 256         if not variant:
 257             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 258             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 259             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 260             embedscreen['priority'] -= 3
 261         elif variant == 'embedded':
 262             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 263             ytcfg['priority'] -= 2
 264         else:
 265             ytcfg['priority'] -= 3
 266
 267
 268 build_innertube_clients()
 269
 270
 271 class YoutubeBaseInfoExtractor(InfoExtractor):
 272     """Provide base functions for Youtube extractors"""
 273
 274     _RESERVED_NAMES = (
 275         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 276         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 277         r'browse|oembed|get_video_info|iframe_api|s/player|'
 278         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 279
 280     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 281
 282     # _NETRC_MACHINE = 'youtube'
 283
 284     # If True it will raise an error if no login info is provided
 285     _LOGIN_REQUIRED = False
 286
 287     _INVIDIOUS_SITES = (
 288         # invidious-redirect websites
 289         r'(?:www\.)?redirect\.invidious\.io',
 290         r'(?:(?:www|dev)\.)?invidio\.us',
 291         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 292         r'(?:www\.)?invidious\.pussthecat\.org',
 293         r'(?:www\.)?invidious\.zee\.li',
 294         r'(?:www\.)?invidious\.ethibox\.fr',
 295         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 296         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 297         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 298         # youtube-dl invidious instances list
 299         r'(?:(?:www|no)\.)?invidiou\.sh',
 300         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 301         r'(?:www\.)?invidious\.kabi\.tk',
 302         r'(?:www\.)?invidious\.mastodon\.host',
 303         r'(?:www\.)?invidious\.zapashcanon\.fr',
 304         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 305         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 306         r'(?:www\.)?invidious\.himiko\.cloud',
 307         r'(?:www\.)?invidious\.reallyancient\.tech',
 308         r'(?:www\.)?invidious\.tube',
 309         r'(?:www\.)?invidiou\.site',
 310         r'(?:www\.)?invidious\.site',
 311         r'(?:www\.)?invidious\.xyz',
 312         r'(?:www\.)?invidious\.nixnet\.xyz',
 313         r'(?:www\.)?invidious\.048596\.xyz',
 314         r'(?:www\.)?invidious\.drycat\.fr',
 315         r'(?:www\.)?inv\.skyn3t\.in',
 316         r'(?:www\.)?tube\.poal\.co',
 317         r'(?:www\.)?tube\.connect\.cafe',
 318         r'(?:www\.)?vid\.wxzm\.sx',
 319         r'(?:www\.)?vid\.mint\.lgbt',
 320         r'(?:www\.)?vid\.puffyan\.us',
 321         r'(?:www\.)?yewtu\.be',
 322         r'(?:www\.)?yt\.elukerio\.org',
 323         r'(?:www\.)?yt\.lelux\.fi',
 324         r'(?:www\.)?invidious\.ggc-project\.de',
 325         r'(?:www\.)?yt\.maisputain\.ovh',
 326         r'(?:www\.)?ytprivate\.com',
 327         r'(?:www\.)?invidious\.13ad\.de',
 328         r'(?:www\.)?invidious\.toot\.koeln',
 329         r'(?:www\.)?invidious\.fdn\.fr',
 330         r'(?:www\.)?watch\.nettohikari\.com',
 331         r'(?:www\.)?invidious\.namazso\.eu',
 332         r'(?:www\.)?invidious\.silkky\.cloud',
 333         r'(?:www\.)?invidious\.exonip\.de',
 334         r'(?:www\.)?invidious\.riverside\.rocks',
 335         r'(?:www\.)?invidious\.blamefran\.net',
 336         r'(?:www\.)?invidious\.moomoo\.de',
 337         r'(?:www\.)?ytb\.trom\.tf',
 338         r'(?:www\.)?yt\.cyberhost\.uk',
 339         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 340         r'(?:www\.)?qklhadlycap4cnod\.onion',
 341         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 342         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 343         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 344         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 345         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 346         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 347         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 348         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 349         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 350         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 351     )
 352
 353     def _initialize_consent(self):
 354         cookies = self._get_cookies('https://www.youtube.com/')
 355         if cookies.get('__Secure-3PSID'):
 356             return
 357         consent_id = None
 358         consent = cookies.get('CONSENT')
 359         if consent:
 360             if 'YES' in consent.value:
 361                 return
 362             consent_id = self._search_regex(
 363                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 364         if not consent_id:
 365             consent_id = random.randint(100, 999)
 366         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 367
 368     def _initialize_pref(self):
 369         cookies = self._get_cookies('https://www.youtube.com/')
 370         pref_cookie = cookies.get('PREF')
 371         pref = {}
 372         if pref_cookie:
 373             try:
 374                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 375             except ValueError:
 376                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 377         pref.update({'hl': 'en', 'tz': 'UTC'})
 378         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 379
 380     def _real_initialize(self):
 381         self._initialize_pref()
 382         self._initialize_consent()
 383         self._check_login_required()
 384
 385     def _check_login_required(self):
 386         if (self._LOGIN_REQUIRED
 387                 and self.get_param('cookiefile') is None
 388                 and self.get_param('cookiesfrombrowser') is None):
 389             self.raise_login_required('Login details are needed to download this content', method='cookies')
 390
 391     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 392     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 393     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 394
 395     def _get_default_ytcfg(self, client='web'):
 396         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 397
 398     def _get_innertube_host(self, client='web'):
 399         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 400
 401     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 402         # try_get but with fallback to default ytcfg client values when present
 403         _func = lambda y: try_get(y, getter, expected_type)
 404         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 405
 406     def _extract_client_name(self, ytcfg, default_client='web'):
 407         return self._ytcfg_get_safe(
 408             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 409                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 410
 411     def _extract_client_version(self, ytcfg, default_client='web'):
 412         return self._ytcfg_get_safe(
 413             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 414                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 415
 416     def _extract_api_key(self, ytcfg=None, default_client='web'):
 417         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 418
 419     def _extract_context(self, ytcfg=None, default_client='web'):
 420         context = get_first(
 421             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 422         # Enforce language and tz for extraction
 423         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 424         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 425         return context
 426
 427     _SAPISID = None
 428
 429     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 430         time_now = round(time.time())
 431         if self._SAPISID is None:
 432             yt_cookies = self._get_cookies('https://www.youtube.com')
 433             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 434             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 435             sapisid_cookie = dict_get(
 436                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 437             if sapisid_cookie and sapisid_cookie.value:
 438                 self._SAPISID = sapisid_cookie.value
 439                 self.write_debug('Extracted SAPISID cookie')
 440                 # SAPISID cookie is required if not already present
 441                 if not yt_cookies.get('SAPISID'):
 442                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 443                     self._set_cookie(
 444                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 445             else:
 446                 self._SAPISID = False
 447         if not self._SAPISID:
 448             return None
 449         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 450         sapisidhash = hashlib.sha1(
 451             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 452         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 453
 454     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 455                   note='Downloading API JSON', errnote='Unable to download API page',
 456                   context=None, api_key=None, api_hostname=None, default_client='web'):
 457
 458         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 459         data.update(query)
 460         real_headers = self.generate_api_headers(default_client=default_client)
 461         real_headers.update({'content-type': 'application/json'})
 462         if headers:
 463             real_headers.update(headers)
 464         return self._download_json(
 465             f'https://{api_hostname or self._get_innertube_host(default_client)}/youtubei/v1/{ep}',
 466             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 467             data=json.dumps(data).encode('utf8'), headers=real_headers,
 468             query={'key': api_key or self._extract_api_key(), 'prettyPrint': 'false'})
 469
 470     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 471         data = self._search_regex(
 472             (fr'{self._YT_INITIAL_DATA_RE}\s*{self._YT_INITIAL_BOUNDARY_RE}',
 473              self._YT_INITIAL_DATA_RE), webpage, 'yt initial data', fatal=fatal)
 474         if data:
 475             return self._parse_json(data, item_id, fatal=fatal)
 476
 477     @staticmethod
 478     def _extract_session_index(*data):
 479         """
 480         Index of current account in account list.
 481         See: https://github.com/yt-dlp/yt-dlp/pull/519
 482         """
 483         for ytcfg in data:
 484             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 485             if session_index is not None:
 486                 return session_index
 487
 488     # Deprecated?
 489     def _extract_identity_token(self, ytcfg=None, webpage=None):
 490         if ytcfg:
 491             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 492             if token:
 493                 return token
 494         if webpage:
 495             return self._search_regex(
 496                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 497                 'identity token', default=None, fatal=False)
 498
 499     @staticmethod
 500     def _extract_account_syncid(*args):
 501         """
 502         Extract syncId required to download private playlists of secondary channels
 503         @params response and/or ytcfg
 504         """
 505         for data in args:
 506             # ytcfg includes channel_syncid if on secondary channel
 507             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 508             if delegated_sid:
 509                 return delegated_sid
 510             sync_ids = (try_get(
 511                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 512                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 513             if len(sync_ids) >= 2 and sync_ids[1]:
 514                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 515                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 516                 return sync_ids[0]
 517
 518     @staticmethod
 519     def _extract_visitor_data(*args):
 520         """
 521         Extracts visitorData from an API response or ytcfg
 522         Appears to be used to track session state
 523         """
 524         return get_first(
 525             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 526             expected_type=str)
 527
 528     @property
 529     def is_authenticated(self):
 530         return bool(self._generate_sapisidhash_header())
 531
 532     def extract_ytcfg(self, video_id, webpage):
 533         if not webpage:
 534             return {}
 535         return self._parse_json(
 536             self._search_regex(
 537                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 538                 default='{}'), video_id, fatal=False) or {}
 539
 540     def generate_api_headers(
 541             self, *, ytcfg=None, account_syncid=None, session_index=None,
 542             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 543
 544         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
 545         headers = {
 546             'X-YouTube-Client-Name': compat_str(
 547                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 548             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 549             'Origin': origin,
 550             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 551             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 552             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 553         }
 554         if session_index is None:
 555             session_index = self._extract_session_index(ytcfg)
 556         if account_syncid or session_index is not None:
 557             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 558
 559         auth = self._generate_sapisidhash_header(origin)
 560         if auth is not None:
 561             headers['Authorization'] = auth
 562             headers['X-Origin'] = origin
 563         return {h: v for h, v in headers.items() if v is not None}
 564
 565     def _download_ytcfg(self, client, video_id):
 566         url = {
 567             'web': 'https://www.youtube.com',
 568             'web_music': 'https://music.youtube.com',
 569             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 570         }.get(client)
 571         if not url:
 572             return {}
 573         webpage = self._download_webpage(
 574             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 575         return self.extract_ytcfg(video_id, webpage) or {}
 576
 577     @staticmethod
 578     def _build_api_continuation_query(continuation, ctp=None):
 579         query = {
 580             'continuation': continuation
 581         }
 582         # TODO: Inconsistency with clickTrackingParams.
 583         # Currently we have a fixed ctp contained within context (from ytcfg)
 584         # and a ctp in root query for continuation.
 585         if ctp:
 586             query['clickTracking'] = {'clickTrackingParams': ctp}
 587         return query
 588
 589     @classmethod
 590     def _extract_next_continuation_data(cls, renderer):
 591         next_continuation = try_get(
 592             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 593                        lambda x: x['continuation']['reloadContinuationData']), dict)
 594         if not next_continuation:
 595             return
 596         continuation = next_continuation.get('continuation')
 597         if not continuation:
 598             return
 599         ctp = next_continuation.get('clickTrackingParams')
 600         return cls._build_api_continuation_query(continuation, ctp)
 601
 602     @classmethod
 603     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 604         if isinstance(continuation_ep, dict):
 605             continuation = try_get(
 606                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 607             if not continuation:
 608                 return
 609             ctp = continuation_ep.get('clickTrackingParams')
 610             return cls._build_api_continuation_query(continuation, ctp)
 611
 612     @classmethod
 613     def _extract_continuation(cls, renderer):
 614         next_continuation = cls._extract_next_continuation_data(renderer)
 615         if next_continuation:
 616             return next_continuation
 617
 618         contents = []
 619         for key in ('contents', 'items'):
 620             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 621
 622         for content in contents:
 623             if not isinstance(content, dict):
 624                 continue
 625             continuation_ep = try_get(
 626                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 627                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 628                 dict)
 629             continuation = cls._extract_continuation_ep_data(continuation_ep)
 630             if continuation:
 631                 return continuation
 632
 633     @classmethod
 634     def _extract_alerts(cls, data):
 635         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 636             if not isinstance(alert_dict, dict):
 637                 continue
 638             for alert in alert_dict.values():
 639                 alert_type = alert.get('type')
 640                 if not alert_type:
 641                     continue
 642                 message = cls._get_text(alert, 'text')
 643                 if message:
 644                     yield alert_type, message
 645
 646     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 647         errors = []
 648         warnings = []
 649         for alert_type, alert_message in alerts:
 650             if alert_type.lower() == 'error' and fatal:
 651                 errors.append([alert_type, alert_message])
 652             else:
 653                 warnings.append([alert_type, alert_message])
 654
 655         for alert_type, alert_message in (warnings + errors[:-1]):
 656             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 657         if errors:
 658             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 659
 660     def _extract_and_report_alerts(self, data, *args, **kwargs):
 661         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 662
 663     def _extract_badges(self, renderer: dict):
 664         badges = set()
 665         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 666             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 667             if label:
 668                 badges.add(label.lower())
 669         return badges
 670
 671     @staticmethod
 672     def _get_text(data, *path_list, max_runs=None):
 673         for path in path_list or [None]:
 674             if path is None:
 675                 obj = [data]
 676             else:
 677                 obj = traverse_obj(data, path, default=[])
 678                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 679                     obj = [obj]
 680             for item in obj:
 681                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 682                 if text:
 683                     return text
 684                 runs = try_get(item, lambda x: x['runs'], list) or []
 685                 if not runs and isinstance(item, list):
 686                     runs = item
 687
 688                 runs = runs[:min(len(runs), max_runs or len(runs))]
 689                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 690                 if text:
 691                     return text
 692
 693     def _get_count(self, data, *path_list):
 694         count_text = self._get_text(data, *path_list) or ''
 695         count = parse_count(count_text)
 696         if count is None:
 697             count = str_to_int(
 698                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 699         return count
 700
 701     @staticmethod
 702     def _extract_thumbnails(data, *path_list):
 703         """
 704         Extract thumbnails from thumbnails dict
 705         @param path_list: path list to level that contains 'thumbnails' key
 706         """
 707         thumbnails = []
 708         for path in path_list or [()]:
 709             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 710                 thumbnail_url = url_or_none(thumbnail.get('url'))
 711                 if not thumbnail_url:
 712                     continue
 713                 # Sometimes youtube gives a wrong thumbnail URL. See:
 714                 # https://github.com/yt-dlp/yt-dlp/issues/233
 715                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 716                 if 'maxresdefault' in thumbnail_url:
 717                     thumbnail_url = thumbnail_url.split('?')[0]
 718                 thumbnails.append({
 719                     'url': thumbnail_url,
 720                     'height': int_or_none(thumbnail.get('height')),
 721                     'width': int_or_none(thumbnail.get('width')),
 722                 })
 723         return thumbnails
 724
 725     @staticmethod
 726     def extract_relative_time(relative_time_text):
 727         """
 728         Extracts a relative time from string and converts to dt object
 729         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 730         """
 731         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 732         if mobj:
 733             start = mobj.group('start')
 734             if start:
 735                 return datetime_from_str(start)
 736             try:
 737                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 738             except ValueError:
 739                 return None
 740
 741     def _extract_time_text(self, renderer, *path_list):
 742         """@returns (timestamp, time_text)"""
 743         text = self._get_text(renderer, *path_list) or ''
 744         dt = self.extract_relative_time(text)
 745         timestamp = None
 746         if isinstance(dt, datetime.datetime):
 747             timestamp = calendar.timegm(dt.timetuple())
 748
 749         if timestamp is None:
 750             timestamp = (
 751                 unified_timestamp(text) or unified_timestamp(
 752                     self._search_regex(
 753                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 754                         text.lower(), 'time text', default=None)))
 755
 756         if text and timestamp is None:
 757             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 758         return timestamp, text
 759
 760     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 761                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 762                           default_client='web'):
 763         response = None
 764         last_error = None
 765         count = -1
 766         retries = self.get_param('extractor_retries', 3)
 767         if check_get_keys is None:
 768             check_get_keys = []
 769         while count < retries:
 770             count += 1
 771             if last_error:
 772                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 773             try:
 774                 response = self._call_api(
 775                     ep=ep, fatal=True, headers=headers,
 776                     video_id=item_id, query=query,
 777                     context=self._extract_context(ytcfg, default_client),
 778                     api_key=self._extract_api_key(ytcfg, default_client),
 779                     api_hostname=api_hostname, default_client=default_client,
 780                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 781             except ExtractorError as e:
 782                 if isinstance(e.cause, network_exceptions):
 783                     if isinstance(e.cause, compat_HTTPError):
 784                         first_bytes = e.cause.read(512)
 785                         if not is_html(first_bytes):
 786                             yt_error = try_get(
 787                                 self._parse_json(
 788                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 789                                 lambda x: x['error']['message'], compat_str)
 790                             if yt_error:
 791                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 792                     # Downloading page may result in intermittent 5xx HTTP error
 793                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 794                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 795                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 796                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 797                         last_error = error_to_compat_str(e.cause or e.msg)
 798                         if count < retries:
 799                             continue
 800                 if fatal:
 801                     raise
 802                 else:
 803                     self.report_warning(error_to_compat_str(e))
 804                     return
 805
 806             else:
 807                 try:
 808                     self._extract_and_report_alerts(response, only_once=True)
 809                 except ExtractorError as e:
 810                     # YouTube servers may return errors we want to retry on in a 200 OK response
 811                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 812                     if 'unknown error' in e.msg.lower():
 813                         last_error = e.msg
 814                         continue
 815                     if fatal:
 816                         raise
 817                     self.report_warning(error_to_compat_str(e))
 818                     return
 819                 if not check_get_keys or dict_get(response, check_get_keys):
 820                     break
 821                 # Youtube sometimes sends incomplete data
 822                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 823                 last_error = 'Incomplete data received'
 824                 if count >= retries:
 825                     if fatal:
 826                         raise ExtractorError(last_error)
 827                     else:
 828                         self.report_warning(last_error)
 829                         return
 830         return response
 831
 832     @staticmethod
 833     def is_music_url(url):
 834         return re.match(r'https?://music\.youtube\.com/', url) is not None
 835
 836     def _extract_video(self, renderer):
 837         video_id = renderer.get('videoId')
 838         title = self._get_text(renderer, 'title')
 839         description = self._get_text(renderer, 'descriptionSnippet')
 840         duration = parse_duration(self._get_text(
 841             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 842         if duration is None:
 843             duration = parse_duration(self._search_regex(
 844                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 845                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 846                 video_id, default=None, group='duration'))
 847
 848         view_count = self._get_count(renderer, 'viewCountText')
 849
 850         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 851         channel_id = traverse_obj(
 852             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 853             expected_type=str, get_all=False)
 854         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 855         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 856         overlay_style = traverse_obj(
 857             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 858             get_all=False, expected_type=str)
 859         badges = self._extract_badges(renderer)
 860         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 861         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 862             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 863             expected_type=str)) or ''
 864         url = f'https://www.youtube.com/watch?v={video_id}'
 865         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 866             url = f'https://www.youtube.com/shorts/{video_id}'
 867
 868         return {
 869             '_type': 'url',
 870             'ie_key': YoutubeIE.ie_key(),
 871             'id': video_id,
 872             'url': url,
 873             'title': title,
 874             'description': description,
 875             'duration': duration,
 876             'view_count': view_count,
 877             'uploader': uploader,
 878             'channel_id': channel_id,
 879             'thumbnails': thumbnails,
 880             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 881                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 882                             else None),
 883             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 884                             else 'was_live' if 'streamed' in time_text.lower()
 885                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 886                             else None),
 887             'release_timestamp': scheduled_timestamp,
 888             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 889         }
 890
 891
 892 class YoutubeIE(YoutubeBaseInfoExtractor):
 893     IE_DESC = 'YouTube'
 894     _VALID_URL = r"""(?x)^
 895                      (
 896                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 897                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 898                             (?:www\.)?deturl\.com/www\.youtube\.com|
 899                             (?:www\.)?pwnyoutube\.com|
 900                             (?:www\.)?hooktube\.com|
 901                             (?:www\.)?yourepeat\.com|
 902                             tube\.majestyc\.net|
 903                             %(invidious)s|
 904                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 905                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 906                          (?:                                                  # the various things that can precede the ID:
 907                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 908                              |(?:                                             # or the v= param in all its forms
 909                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 910                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 911                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 912                                  v=
 913                              )
 914                          ))
 915                          |(?:
 916                             youtu\.be|                                        # just youtu.be/xxxx
 917                             vid\.plus|                                        # or vid.plus/xxxx
 918                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 919                             %(invidious)s
 920                          )/
 921                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 922                          )
 923                      )?                                                       # all until now is optional -> you can pass the naked ID
 924                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 925                      (?(1).+)?                                                # if we found the ID, everything can follow
 926                      (?:\#|$)""" % {
 927         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 928     }
 929     _PLAYER_INFO_RE = (
 930         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 931         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 932         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 933     )
 934     _formats = {
 935         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 936         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 937         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 938         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 939         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 940         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 941         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 942         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 943         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 944         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 945         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 946         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 947         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 948         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 949         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 950         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 951         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 953
 954
 955         # 3D videos
 956         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 957         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 958         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 959         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 960         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 961         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 962         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 963
 964         # Apple HTTP Live Streaming
 965         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 966         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 967         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 968         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 969         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 970         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 971         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 972         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 973
 974         # DASH mp4 video
 975         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 976         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 977         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 978         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 979         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 980         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 981         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 982         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 983         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 984         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 985         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 986         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987
 988         # Dash mp4 audio
 989         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 990         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 991         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 992         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 993         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 994         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 995         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 996
 997         # Dash webm
 998         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 999         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1000         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1001         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1002         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1003         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1004         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1005         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1006         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1007         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1008         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1009         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1010         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1011         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1012         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1013         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1014         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1015         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1016         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1017         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1018         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1020
1021         # Dash webm audio
1022         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1023         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1024
1025         # Dash webm audio with opus inside
1026         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1027         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1028         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1029
1030         # RTMP (unnamed)
1031         '_rtmp': {'protocol': 'rtmp'},
1032
1033         # av01 video only formats sometimes served with "unknown" codecs
1034         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1035         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1036         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1037         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1038         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1039         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1040         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1041         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1042     }
1043     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1044
1045     _GEO_BYPASS = False
1046
1047     IE_NAME = 'youtube'
1048     _TESTS = [
1049         {
1050             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1051             'info_dict': {
1052                 'id': 'BaW_jenozKc',
1053                 'ext': 'mp4',
1054                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1055                 'uploader': 'Philipp Hagemeister',
1056                 'uploader_id': 'phihag',
1057                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1058                 'channel': 'Philipp Hagemeister',
1059                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1060                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1061                 'upload_date': '20121002',
1062                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1063                 'categories': ['Science & Technology'],
1064                 'tags': ['youtube-dl'],
1065                 'duration': 10,
1066                 'view_count': int,
1067                 'like_count': int,
1068                 'availability': 'public',
1069                 'playable_in_embed': True,
1070                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1071                 'live_status': 'not_live',
1072                 'age_limit': 0,
1073                 'start_time': 1,
1074                 'end_time': 9,
1075                 'channel_follower_count': int
1076             }
1077         },
1078         {
1079             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1080             'note': 'Embed-only video (#1746)',
1081             'info_dict': {
1082                 'id': 'yZIXLfi8CZQ',
1083                 'ext': 'mp4',
1084                 'upload_date': '20120608',
1085                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1086                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1087                 'uploader': 'SET India',
1088                 'uploader_id': 'setindia',
1089                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1090                 'age_limit': 18,
1091             },
1092             'skip': 'Private video',
1093         },
1094         {
1095             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1096             'note': 'Use the first video ID in the URL',
1097             'info_dict': {
1098                 'id': 'BaW_jenozKc',
1099                 'ext': 'mp4',
1100                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1101                 'uploader': 'Philipp Hagemeister',
1102                 'uploader_id': 'phihag',
1103                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1104                 'channel': 'Philipp Hagemeister',
1105                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1106                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1107                 'upload_date': '20121002',
1108                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1109                 'categories': ['Science & Technology'],
1110                 'tags': ['youtube-dl'],
1111                 'duration': 10,
1112                 'view_count': int,
1113                 'like_count': int,
1114                 'availability': 'public',
1115                 'playable_in_embed': True,
1116                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1117                 'live_status': 'not_live',
1118                 'age_limit': 0,
1119                 'channel_follower_count': int
1120             },
1121             'params': {
1122                 'skip_download': True,
1123             },
1124         },
1125         {
1126             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1127             'note': '256k DASH audio (format 141) via DASH manifest',
1128             'info_dict': {
1129                 'id': 'a9LDPn-MO4I',
1130                 'ext': 'm4a',
1131                 'upload_date': '20121002',
1132                 'uploader_id': '8KVIDEO',
1133                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1134                 'description': '',
1135                 'uploader': '8KVIDEO',
1136                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1137             },
1138             'params': {
1139                 'youtube_include_dash_manifest': True,
1140                 'format': '141',
1141             },
1142             'skip': 'format 141 not served anymore',
1143         },
1144         # DASH manifest with encrypted signature
1145         {
1146             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1147             'info_dict': {
1148                 'id': 'IB3lcPjvWLA',
1149                 'ext': 'm4a',
1150                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1151                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1152                 'duration': 244,
1153                 'uploader': 'AfrojackVEVO',
1154                 'uploader_id': 'AfrojackVEVO',
1155                 'upload_date': '20131011',
1156                 'abr': 129.495,
1157                 'like_count': int,
1158                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1159                 'playable_in_embed': True,
1160                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1161                 'view_count': int,
1162                 'track': 'The Spark',
1163                 'live_status': 'not_live',
1164                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1165                 'channel': 'Afrojack',
1166                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1167                 'tags': 'count:19',
1168                 'availability': 'public',
1169                 'categories': ['Music'],
1170                 'age_limit': 0,
1171                 'alt_title': 'The Spark',
1172                 'channel_follower_count': int
1173             },
1174             'params': {
1175                 'youtube_include_dash_manifest': True,
1176                 'format': '141/bestaudio[ext=m4a]',
1177             },
1178         },
1179         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1180         {
1181             'note': 'Embed allowed age-gate video',
1182             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1183             'info_dict': {
1184                 'id': 'HtVdAasjOgU',
1185                 'ext': 'mp4',
1186                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1187                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1188                 'duration': 142,
1189                 'uploader': 'The Witcher',
1190                 'uploader_id': 'WitcherGame',
1191                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1192                 'upload_date': '20140605',
1193                 'age_limit': 18,
1194                 'categories': ['Gaming'],
1195                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1196                 'availability': 'needs_auth',
1197                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1198                 'like_count': int,
1199                 'channel': 'The Witcher',
1200                 'live_status': 'not_live',
1201                 'tags': 'count:17',
1202                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1203                 'playable_in_embed': True,
1204                 'view_count': int,
1205                 'channel_follower_count': int
1206             },
1207         },
1208         {
1209             'note': 'Age-gate video with embed allowed in public site',
1210             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1211             'info_dict': {
1212                 'id': 'HsUATh_Nc2U',
1213                 'ext': 'mp4',
1214                 'title': 'Godzilla 2 (Official Video)',
1215                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1216                 'upload_date': '20200408',
1217                 'uploader_id': 'FlyingKitty900',
1218                 'uploader': 'FlyingKitty',
1219                 'age_limit': 18,
1220                 'availability': 'needs_auth',
1221                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1222                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1223                 'channel': 'FlyingKitty',
1224                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1225                 'view_count': int,
1226                 'categories': ['Entertainment'],
1227                 'live_status': 'not_live',
1228                 'tags': ['Flyingkitty', 'godzilla 2'],
1229                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1230                 'like_count': int,
1231                 'duration': 177,
1232                 'playable_in_embed': True,
1233                 'channel_follower_count': int
1234             },
1235         },
1236         {
1237             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1238             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1239             'info_dict': {
1240                 'id': 'Tq92D6wQ1mg',
1241                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1242                 'ext': 'mp4',
1243                 'upload_date': '20191228',
1244                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1245                 'uploader': 'Projekt Melody',
1246                 'description': 'md5:17eccca93a786d51bc67646756894066',
1247                 'age_limit': 18,
1248                 'like_count': int,
1249                 'availability': 'needs_auth',
1250                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1251                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1252                 'view_count': int,
1253                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1254                 'channel': 'Projekt Melody',
1255                 'live_status': 'not_live',
1256                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1257                 'playable_in_embed': True,
1258                 'categories': ['Entertainment'],
1259                 'duration': 106,
1260                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1261                 'channel_follower_count': int
1262             },
1263         },
1264         {
1265             'note': 'Non-Agegated non-embeddable video',
1266             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1267             'info_dict': {
1268                 'id': 'MeJVWBSsPAY',
1269                 'ext': 'mp4',
1270                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1271                 'uploader': 'Herr Lurik',
1272                 'uploader_id': 'st3in234',
1273                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1274                 'upload_date': '20130730',
1275                 'track': 'Such mich find mich',
1276                 'age_limit': 0,
1277                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1278                 'like_count': int,
1279                 'playable_in_embed': False,
1280                 'creator': 'OOMPH!',
1281                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1282                 'view_count': int,
1283                 'alt_title': 'Such mich find mich',
1284                 'duration': 210,
1285                 'channel': 'Herr Lurik',
1286                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1287                 'categories': ['Music'],
1288                 'availability': 'public',
1289                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1290                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1291                 'live_status': 'not_live',
1292                 'artist': 'OOMPH!',
1293                 'channel_follower_count': int
1294             },
1295         },
1296         {
1297             'note': 'Non-bypassable age-gated video',
1298             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1299             'only_matching': True,
1300         },
1301         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1302         # YouTube Red ad is not captured for creator
1303         {
1304             'url': '__2ABJjxzNo',
1305             'info_dict': {
1306                 'id': '__2ABJjxzNo',
1307                 'ext': 'mp4',
1308                 'duration': 266,
1309                 'upload_date': '20100430',
1310                 'uploader_id': 'deadmau5',
1311                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1312                 'creator': 'deadmau5',
1313                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1314                 'uploader': 'deadmau5',
1315                 'title': 'Deadmau5 - Some Chords (HD)',
1316                 'alt_title': 'Some Chords',
1317                 'availability': 'public',
1318                 'tags': 'count:14',
1319                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1320                 'view_count': int,
1321                 'live_status': 'not_live',
1322                 'channel': 'deadmau5',
1323                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1324                 'like_count': int,
1325                 'track': 'Some Chords',
1326                 'artist': 'deadmau5',
1327                 'playable_in_embed': True,
1328                 'age_limit': 0,
1329                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1330                 'categories': ['Music'],
1331                 'album': 'Some Chords',
1332                 'channel_follower_count': int
1333             },
1334             'expected_warnings': [
1335                 'DASH manifest missing',
1336             ]
1337         },
1338         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1339         {
1340             'url': 'lqQg6PlCWgI',
1341             'info_dict': {
1342                 'id': 'lqQg6PlCWgI',
1343                 'ext': 'mp4',
1344                 'duration': 6085,
1345                 'upload_date': '20150827',
1346                 'uploader_id': 'olympic',
1347                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1348                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1349                 'uploader': 'Olympics',
1350                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1351                 'like_count': int,
1352                 'release_timestamp': 1343767800,
1353                 'playable_in_embed': True,
1354                 'categories': ['Sports'],
1355                 'release_date': '20120731',
1356                 'channel': 'Olympics',
1357                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1358                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1359                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1360                 'age_limit': 0,
1361                 'availability': 'public',
1362                 'live_status': 'was_live',
1363                 'view_count': int,
1364                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1365                 'channel_follower_count': int
1366             },
1367             'params': {
1368                 'skip_download': 'requires avconv',
1369             }
1370         },
1371         # Non-square pixels
1372         {
1373             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1374             'info_dict': {
1375                 'id': '_b-2C3KPAM0',
1376                 'ext': 'mp4',
1377                 'stretched_ratio': 16 / 9.,
1378                 'duration': 85,
1379                 'upload_date': '20110310',
1380                 'uploader_id': 'AllenMeow',
1381                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1382                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1383                 'uploader': '孫ᄋᄅ',
1384                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1385                 'playable_in_embed': True,
1386                 'channel': '孫ᄋᄅ',
1387                 'age_limit': 0,
1388                 'tags': 'count:11',
1389                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1390                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1391                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1392                 'view_count': int,
1393                 'categories': ['People & Blogs'],
1394                 'like_count': int,
1395                 'live_status': 'not_live',
1396                 'availability': 'unlisted',
1397                 'channel_follower_count': int
1398             },
1399         },
1400         # url_encoded_fmt_stream_map is empty string
1401         {
1402             'url': 'qEJwOuvDf7I',
1403             'info_dict': {
1404                 'id': 'qEJwOuvDf7I',
1405                 'ext': 'webm',
1406                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1407                 'description': '',
1408                 'upload_date': '20150404',
1409                 'uploader_id': 'spbelect',
1410                 'uploader': 'Наблюдатели Петербурга',
1411             },
1412             'params': {
1413                 'skip_download': 'requires avconv',
1414             },
1415             'skip': 'This live event has ended.',
1416         },
1417         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1418         {
1419             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1420             'info_dict': {
1421                 'id': 'FIl7x6_3R5Y',
1422                 'ext': 'webm',
1423                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1424                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1425                 'duration': 220,
1426                 'upload_date': '20150625',
1427                 'uploader_id': 'dorappi2000',
1428                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1429                 'uploader': 'dorappi2000',
1430                 'formats': 'mincount:31',
1431             },
1432             'skip': 'not actual anymore',
1433         },
1434         # DASH manifest with segment_list
1435         {
1436             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1437             'md5': '8ce563a1d667b599d21064e982ab9e31',
1438             'info_dict': {
1439                 'id': 'CsmdDsKjzN8',
1440                 'ext': 'mp4',
1441                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1442                 'uploader': 'Airtek',
1443                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1444                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1445                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1446             },
1447             'params': {
1448                 'youtube_include_dash_manifest': True,
1449                 'format': '135',  # bestvideo
1450             },
1451             'skip': 'This live event has ended.',
1452         },
1453         {
1454             # Multifeed videos (multiple cameras), URL is for Main Camera
1455             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1456             'info_dict': {
1457                 'id': 'jvGDaLqkpTg',
1458                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1459                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1460             },
1461             'playlist': [{
1462                 'info_dict': {
1463                     'id': 'jvGDaLqkpTg',
1464                     'ext': 'mp4',
1465                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1466                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1467                     'duration': 10643,
1468                     'upload_date': '20161111',
1469                     'uploader': 'Team PGP',
1470                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1471                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1472                 },
1473             }, {
1474                 'info_dict': {
1475                     'id': '3AKt1R1aDnw',
1476                     'ext': 'mp4',
1477                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1478                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1479                     'duration': 10991,
1480                     'upload_date': '20161111',
1481                     'uploader': 'Team PGP',
1482                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1483                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1484                 },
1485             }, {
1486                 'info_dict': {
1487                     'id': 'RtAMM00gpVc',
1488                     'ext': 'mp4',
1489                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1490                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1491                     'duration': 10995,
1492                     'upload_date': '20161111',
1493                     'uploader': 'Team PGP',
1494                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1495                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1496                 },
1497             }, {
1498                 'info_dict': {
1499                     'id': '6N2fdlP3C5U',
1500                     'ext': 'mp4',
1501                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1502                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1503                     'duration': 10990,
1504                     'upload_date': '20161111',
1505                     'uploader': 'Team PGP',
1506                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1507                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1508                 },
1509             }],
1510             'params': {
1511                 'skip_download': True,
1512             },
1513             'skip': 'Not multifeed anymore',
1514         },
1515         {
1516             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1517             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1518             'info_dict': {
1519                 'id': 'gVfLd0zydlo',
1520                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1521             },
1522             'playlist_count': 2,
1523             'skip': 'Not multifeed anymore',
1524         },
1525         {
1526             'url': 'https://vid.plus/FlRa-iH7PGw',
1527             'only_matching': True,
1528         },
1529         {
1530             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1531             'only_matching': True,
1532         },
1533         {
1534             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1535             # Also tests cut-off URL expansion in video description (see
1536             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1537             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1538             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1539             'info_dict': {
1540                 'id': 'lsguqyKfVQg',
1541                 'ext': 'mp4',
1542                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1543                 'alt_title': 'Dark Walk',
1544                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1545                 'duration': 133,
1546                 'upload_date': '20151119',
1547                 'uploader_id': 'IronSoulElf',
1548                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1549                 'uploader': 'IronSoulElf',
1550                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1551                 'track': 'Dark Walk',
1552                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1553                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1554                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1555                 'categories': ['Film & Animation'],
1556                 'view_count': int,
1557                 'live_status': 'not_live',
1558                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1559                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1560                 'tags': 'count:13',
1561                 'availability': 'public',
1562                 'channel': 'IronSoulElf',
1563                 'playable_in_embed': True,
1564                 'like_count': int,
1565                 'age_limit': 0,
1566                 'channel_follower_count': int
1567             },
1568             'params': {
1569                 'skip_download': True,
1570             },
1571         },
1572         {
1573             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1574             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1575             'only_matching': True,
1576         },
1577         {
1578             # Video with yt:stretch=17:0
1579             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1580             'info_dict': {
1581                 'id': 'Q39EVAstoRM',
1582                 'ext': 'mp4',
1583                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1584                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1585                 'upload_date': '20151107',
1586                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1587                 'uploader': 'CH GAMER DROID',
1588             },
1589             'params': {
1590                 'skip_download': True,
1591             },
1592             'skip': 'This video does not exist.',
1593         },
1594         {
1595             # Video with incomplete 'yt:stretch=16:'
1596             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1597             'only_matching': True,
1598         },
1599         {
1600             # Video licensed under Creative Commons
1601             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1602             'info_dict': {
1603                 'id': 'M4gD1WSo5mA',
1604                 'ext': 'mp4',
1605                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1606                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1607                 'duration': 721,
1608                 'upload_date': '20150128',
1609                 'uploader_id': 'BerkmanCenter',
1610                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1611                 'uploader': 'The Berkman Klein Center for Internet & Society',
1612                 'license': 'Creative Commons Attribution license (reuse allowed)',
1613                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1614                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1615                 'like_count': int,
1616                 'age_limit': 0,
1617                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1618                 'channel': 'The Berkman Klein Center for Internet & Society',
1619                 'availability': 'public',
1620                 'view_count': int,
1621                 'categories': ['Education'],
1622                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1623                 'live_status': 'not_live',
1624                 'playable_in_embed': True,
1625                 'channel_follower_count': int
1626             },
1627             'params': {
1628                 'skip_download': True,
1629             },
1630         },
1631         {
1632             # Channel-like uploader_url
1633             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1634             'info_dict': {
1635                 'id': 'eQcmzGIKrzg',
1636                 'ext': 'mp4',
1637                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1638                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1639                 'duration': 4060,
1640                 'upload_date': '20151120',
1641                 'uploader': 'Bernie Sanders',
1642                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1643                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1644                 'license': 'Creative Commons Attribution license (reuse allowed)',
1645                 'playable_in_embed': True,
1646                 'tags': 'count:12',
1647                 'like_count': int,
1648                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1649                 'age_limit': 0,
1650                 'availability': 'public',
1651                 'categories': ['News & Politics'],
1652                 'channel': 'Bernie Sanders',
1653                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1654                 'view_count': int,
1655                 'live_status': 'not_live',
1656                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1657                 'channel_follower_count': int
1658             },
1659             'params': {
1660                 'skip_download': True,
1661             },
1662         },
1663         {
1664             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1665             'only_matching': True,
1666         },
1667         {
1668             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1669             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1670             'only_matching': True,
1671         },
1672         {
1673             # Rental video preview
1674             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1675             'info_dict': {
1676                 'id': 'uGpuVWrhIzE',
1677                 'ext': 'mp4',
1678                 'title': 'Piku - Trailer',
1679                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1680                 'upload_date': '20150811',
1681                 'uploader': 'FlixMatrix',
1682                 'uploader_id': 'FlixMatrixKaravan',
1683                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1684                 'license': 'Standard YouTube License',
1685             },
1686             'params': {
1687                 'skip_download': True,
1688             },
1689             'skip': 'This video is not available.',
1690         },
1691         {
1692             # YouTube Red video with episode data
1693             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1694             'info_dict': {
1695                 'id': 'iqKdEhx-dD4',
1696                 'ext': 'mp4',
1697                 'title': 'Isolation - Mind Field (Ep 1)',
1698                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1699                 'duration': 2085,
1700                 'upload_date': '20170118',
1701                 'uploader': 'Vsauce',
1702                 'uploader_id': 'Vsauce',
1703                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1704                 'series': 'Mind Field',
1705                 'season_number': 1,
1706                 'episode_number': 1,
1707                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1708                 'tags': 'count:12',
1709                 'view_count': int,
1710                 'availability': 'public',
1711                 'age_limit': 0,
1712                 'channel': 'Vsauce',
1713                 'episode': 'Episode 1',
1714                 'categories': ['Entertainment'],
1715                 'season': 'Season 1',
1716                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1717                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1718                 'like_count': int,
1719                 'playable_in_embed': True,
1720                 'live_status': 'not_live',
1721                 'channel_follower_count': int
1722             },
1723             'params': {
1724                 'skip_download': True,
1725             },
1726             'expected_warnings': [
1727                 'Skipping DASH manifest',
1728             ],
1729         },
1730         {
1731             # The following content has been identified by the YouTube community
1732             # as inappropriate or offensive to some audiences.
1733             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1734             'info_dict': {
1735                 'id': '6SJNVb0GnPI',
1736                 'ext': 'mp4',
1737                 'title': 'Race Differences in Intelligence',
1738                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1739                 'duration': 965,
1740                 'upload_date': '20140124',
1741                 'uploader': 'New Century Foundation',
1742                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1743                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1744             },
1745             'params': {
1746                 'skip_download': True,
1747             },
1748             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1749         },
1750         {
1751             # itag 212
1752             'url': '1t24XAntNCY',
1753             'only_matching': True,
1754         },
1755         {
1756             # geo restricted to JP
1757             'url': 'sJL6WA-aGkQ',
1758             'only_matching': True,
1759         },
1760         {
1761             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1762             'only_matching': True,
1763         },
1764         {
1765             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1766             'only_matching': True,
1767         },
1768         {
1769             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1770             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1771             'only_matching': True,
1772         },
1773         {
1774             # DRM protected
1775             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1776             'only_matching': True,
1777         },
1778         {
1779             # Video with unsupported adaptive stream type formats
1780             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1781             'info_dict': {
1782                 'id': 'Z4Vy8R84T1U',
1783                 'ext': 'mp4',
1784                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1785                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1786                 'duration': 433,
1787                 'upload_date': '20130923',
1788                 'uploader': 'Amelia Putri Harwita',
1789                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1790                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1791                 'formats': 'maxcount:10',
1792             },
1793             'params': {
1794                 'skip_download': True,
1795                 'youtube_include_dash_manifest': False,
1796             },
1797             'skip': 'not actual anymore',
1798         },
1799         {
1800             # Youtube Music Auto-generated description
1801             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1802             'info_dict': {
1803                 'id': 'MgNrAu2pzNs',
1804                 'ext': 'mp4',
1805                 'title': 'Voyeur Girl',
1806                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1807                 'upload_date': '20190312',
1808                 'uploader': 'Stephen - Topic',
1809                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1810                 'artist': 'Stephen',
1811                 'track': 'Voyeur Girl',
1812                 'album': 'it\'s too much love to know my dear',
1813                 'release_date': '20190313',
1814                 'release_year': 2019,
1815                 'alt_title': 'Voyeur Girl',
1816                 'view_count': int,
1817                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1818                 'playable_in_embed': True,
1819                 'like_count': int,
1820                 'categories': ['Music'],
1821                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1822                 'channel': 'Stephen',
1823                 'availability': 'public',
1824                 'creator': 'Stephen',
1825                 'duration': 169,
1826                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1827                 'age_limit': 0,
1828                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1829                 'tags': 'count:11',
1830                 'live_status': 'not_live',
1831                 'channel_follower_count': int
1832             },
1833             'params': {
1834                 'skip_download': True,
1835             },
1836         },
1837         {
1838             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1839             'only_matching': True,
1840         },
1841         {
1842             # invalid -> valid video id redirection
1843             'url': 'DJztXj2GPfl',
1844             'info_dict': {
1845                 'id': 'DJztXj2GPfk',
1846                 'ext': 'mp4',
1847                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1848                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1849                 'upload_date': '20090125',
1850                 'uploader': 'Prochorowka',
1851                 'uploader_id': 'Prochorowka',
1852                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1853                 'artist': 'Panjabi MC',
1854                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1855                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1856             },
1857             'params': {
1858                 'skip_download': True,
1859             },
1860             'skip': 'Video unavailable',
1861         },
1862         {
1863             # empty description results in an empty string
1864             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1865             'info_dict': {
1866                 'id': 'x41yOUIvK2k',
1867                 'ext': 'mp4',
1868                 'title': 'IMG 3456',
1869                 'description': '',
1870                 'upload_date': '20170613',
1871                 'uploader_id': 'ElevageOrVert',
1872                 'uploader': 'ElevageOrVert',
1873                 'view_count': int,
1874                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1875                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1876                 'like_count': int,
1877                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1878                 'tags': [],
1879                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1880                 'availability': 'public',
1881                 'age_limit': 0,
1882                 'categories': ['Pets & Animals'],
1883                 'duration': 7,
1884                 'playable_in_embed': True,
1885                 'live_status': 'not_live',
1886                 'channel': 'ElevageOrVert',
1887                 'channel_follower_count': int
1888             },
1889             'params': {
1890                 'skip_download': True,
1891             },
1892         },
1893         {
1894             # with '};' inside yt initial data (see [1])
1895             # see [2] for an example with '};' inside ytInitialPlayerResponse
1896             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1897             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1898             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1899             'info_dict': {
1900                 'id': 'CHqg6qOn4no',
1901                 'ext': 'mp4',
1902                 'title': 'Part 77   Sort a list of simple types in c#',
1903                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1904                 'upload_date': '20130831',
1905                 'uploader_id': 'kudvenkat',
1906                 'uploader': 'kudvenkat',
1907                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1908                 'like_count': int,
1909                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1910                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1911                 'live_status': 'not_live',
1912                 'categories': ['Education'],
1913                 'availability': 'public',
1914                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1915                 'tags': 'count:12',
1916                 'playable_in_embed': True,
1917                 'age_limit': 0,
1918                 'view_count': int,
1919                 'duration': 522,
1920                 'channel': 'kudvenkat',
1921                 'channel_follower_count': int
1922             },
1923             'params': {
1924                 'skip_download': True,
1925             },
1926         },
1927         {
1928             # another example of '};' in ytInitialData
1929             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1930             'only_matching': True,
1931         },
1932         {
1933             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1934             'only_matching': True,
1935         },
1936         {
1937             # https://github.com/ytdl-org/youtube-dl/pull/28094
1938             'url': 'OtqTfy26tG0',
1939             'info_dict': {
1940                 'id': 'OtqTfy26tG0',
1941                 'ext': 'mp4',
1942                 'title': 'Burn Out',
1943                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1944                 'upload_date': '20141120',
1945                 'uploader': 'The Cinematic Orchestra - Topic',
1946                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1947                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1948                 'artist': 'The Cinematic Orchestra',
1949                 'track': 'Burn Out',
1950                 'album': 'Every Day',
1951                 'like_count': int,
1952                 'live_status': 'not_live',
1953                 'alt_title': 'Burn Out',
1954                 'duration': 614,
1955                 'age_limit': 0,
1956                 'view_count': int,
1957                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1958                 'creator': 'The Cinematic Orchestra',
1959                 'channel': 'The Cinematic Orchestra',
1960                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1961                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1962                 'availability': 'public',
1963                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1964                 'categories': ['Music'],
1965                 'playable_in_embed': True,
1966                 'channel_follower_count': int
1967             },
1968             'params': {
1969                 'skip_download': True,
1970             },
1971         },
1972         {
1973             # controversial video, only works with bpctr when authenticated with cookies
1974             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1975             'only_matching': True,
1976         },
1977         {
1978             # controversial video, requires bpctr/contentCheckOk
1979             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1980             'info_dict': {
1981                 'id': 'SZJvDhaSDnc',
1982                 'ext': 'mp4',
1983                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1984                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1985                 'uploader': 'CBS Mornings',
1986                 'uploader_id': 'CBSThisMorning',
1987                 'upload_date': '20140716',
1988                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1989                 'duration': 170,
1990                 'categories': ['News & Politics'],
1991                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
1992                 'view_count': int,
1993                 'channel': 'CBS Mornings',
1994                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
1995                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
1996                 'age_limit': 18,
1997                 'availability': 'needs_auth',
1998                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
1999                 'like_count': int,
2000                 'live_status': 'not_live',
2001                 'playable_in_embed': True,
2002                 'channel_follower_count': int
2003             }
2004         },
2005         {
2006             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2007             'url': 'cBvYw8_A0vQ',
2008             'info_dict': {
2009                 'id': 'cBvYw8_A0vQ',
2010                 'ext': 'mp4',
2011                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2012                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2013                 'upload_date': '20201120',
2014                 'uploader': 'Walk around Japan',
2015                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2016                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2017                 'duration': 1456,
2018                 'categories': ['Travel & Events'],
2019                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2020                 'view_count': int,
2021                 'channel': 'Walk around Japan',
2022                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2023                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2024                 'age_limit': 0,
2025                 'availability': 'public',
2026                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2027                 'live_status': 'not_live',
2028                 'playable_in_embed': True,
2029                 'channel_follower_count': int
2030             },
2031             'params': {
2032                 'skip_download': True,
2033             },
2034         }, {
2035             # Has multiple audio streams
2036             'url': 'WaOKSUlf4TM',
2037             'only_matching': True
2038         }, {
2039             # Requires Premium: has format 141 when requested using YTM url
2040             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2041             'only_matching': True
2042         }, {
2043             # multiple subtitles with same lang_code
2044             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2045             'only_matching': True,
2046         }, {
2047             # Force use android client fallback
2048             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2049             'info_dict': {
2050                 'id': 'YOelRv7fMxY',
2051                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2052                 'ext': '3gp',
2053                 'upload_date': '20210624',
2054                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2055                 'uploader': 'colinfurze',
2056                 'uploader_id': 'colinfurze',
2057                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2058                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2059                 'duration': 596,
2060                 'categories': ['Entertainment'],
2061                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2062                 'view_count': int,
2063                 'channel': 'colinfurze',
2064                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2065                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2066                 'age_limit': 0,
2067                 'availability': 'public',
2068                 'like_count': int,
2069                 'live_status': 'not_live',
2070                 'playable_in_embed': True,
2071                 'channel_follower_count': int
2072             },
2073             'params': {
2074                 'format': '17',  # 3gp format available on android
2075                 'extractor_args': {'youtube': {'player_client': ['android']}},
2076             },
2077         },
2078         {
2079             # Skip download of additional client configs (remix client config in this case)
2080             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2081             'only_matching': True,
2082             'params': {
2083                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2084             },
2085         }, {
2086             # shorts
2087             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2088             'only_matching': True,
2089         }, {
2090             'note': 'Storyboards',
2091             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2092             'info_dict': {
2093                 'id': '5KLPxDtMqe8',
2094                 'ext': 'mhtml',
2095                 'format_id': 'sb0',
2096                 'title': 'Your Brain is Plastic',
2097                 'uploader_id': 'scishow',
2098                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2099                 'upload_date': '20140324',
2100                 'uploader': 'SciShow',
2101                 'like_count': int,
2102                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2103                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2104                 'view_count': int,
2105                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2106                 'playable_in_embed': True,
2107                 'tags': 'count:12',
2108                 'uploader_url': 'http://www.youtube.com/user/scishow',
2109                 'availability': 'public',
2110                 'channel': 'SciShow',
2111                 'live_status': 'not_live',
2112                 'duration': 248,
2113                 'categories': ['Education'],
2114                 'age_limit': 0,
2115                 'channel_follower_count': int
2116             }, 'params': {'format': 'mhtml', 'skip_download': True}
2117         }, {
2118             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2119             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2120             'info_dict': {
2121                 'id': '2NUZ8W2llS4',
2122                 'ext': 'mp4',
2123                 'title': 'The NP that test your phone performance 🙂',
2124                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2125                 'uploader': 'Leon Nguyen',
2126                 'uploader_id': 'VNSXIII',
2127                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2128                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2129                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2130                 'duration': 21,
2131                 'view_count': int,
2132                 'age_limit': 0,
2133                 'categories': ['Gaming'],
2134                 'tags': 'count:23',
2135                 'playable_in_embed': True,
2136                 'live_status': 'not_live',
2137                 'upload_date': '20220103',
2138                 'like_count': int,
2139                 'availability': 'public',
2140                 'channel': 'Leon Nguyen',
2141                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2142                 'channel_follower_count': int
2143             }
2144         }, {
2145             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2146             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2147             'info_dict': {
2148                 'id': 'mzZzzBU6lrM',
2149                 'ext': 'mp4',
2150                 'title': 'I Met GeorgeNotFound In Real Life...',
2151                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2152                 'uploader': 'Quackity',
2153                 'uploader_id': 'QuackityHQ',
2154                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2155                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2156                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2157                 'duration': 955,
2158                 'view_count': int,
2159                 'age_limit': 0,
2160                 'categories': ['Entertainment'],
2161                 'tags': 'count:26',
2162                 'playable_in_embed': True,
2163                 'live_status': 'not_live',
2164                 'release_timestamp': 1641172509,
2165                 'release_date': '20220103',
2166                 'upload_date': '20220103',
2167                 'like_count': int,
2168                 'availability': 'public',
2169                 'channel': 'Quackity',
2170                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2171                 'channel_follower_count': int
2172             }
2173         },
2174         {   # continuous livestream. Microformat upload date should be preferred.
2175             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2176             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2177             'info_dict': {
2178                 'id': 'kgx4WGK0oNU',
2179                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2180                 'ext': 'mp4',
2181                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2182                 'availability': 'public',
2183                 'age_limit': 0,
2184                 'release_timestamp': 1637975704,
2185                 'upload_date': '20210619',
2186                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2187                 'live_status': 'is_live',
2188                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2189                 'uploader': '阿鲍Abao',
2190                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2191                 'channel': 'Abao in Tokyo',
2192                 'channel_follower_count': int,
2193                 'release_date': '20211127',
2194                 'tags': 'count:39',
2195                 'categories': ['People & Blogs'],
2196                 'like_count': int,
2197                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2198                 'view_count': int,
2199                 'playable_in_embed': True,
2200                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2201             },
2202             'params': {'skip_download': True}
2203         }, {
2204             # Story. Requires specific player params to work.
2205             # Note: stories get removed after some period of time
2206             'url': 'https://www.youtube.com/watch?v=yN3x1t3sieA',
2207             'info_dict': {
2208                 'id': 'yN3x1t3sieA',
2209                 'ext': 'mp4',
2210                 'uploader': 'Linus Tech Tips',
2211                 'duration': 13,
2212                 'channel': 'Linus Tech Tips',
2213                 'playable_in_embed': True,
2214                 'tags': [],
2215                 'age_limit': 0,
2216                 'uploader_url': 'http://www.youtube.com/user/LinusTechTips',
2217                 'upload_date': '20220402',
2218                 'thumbnail': 'https://i.ytimg.com/vi_webp/yN3x1t3sieA/maxresdefault.webp',
2219                 'title': 'Story',
2220                 'live_status': 'not_live',
2221                 'uploader_id': 'LinusTechTips',
2222                 'view_count': int,
2223                 'description': '',
2224                 'channel_id': 'UCXuqSBlHAE6Xw-yeJA0Tunw',
2225                 'categories': ['Science & Technology'],
2226                 'channel_url': 'https://www.youtube.com/channel/UCXuqSBlHAE6Xw-yeJA0Tunw',
2227                 'availability': 'unlisted',
2228             }
2229         }
2230     ]
2231
2232     @classmethod
2233     def suitable(cls, url):
2234         from ..utils import parse_qs
2235
2236         qs = parse_qs(url)
2237         if qs.get('list', [None])[0]:
2238             return False
2239         return super().suitable(url)
2240
2241     def __init__(self, *args, **kwargs):
2242         super().__init__(*args, **kwargs)
2243         self._code_cache = {}
2244         self._player_cache = {}
2245
2246     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2247         lock = threading.Lock()
2248
2249         is_live = True
2250         start_time = time.time()
2251         formats = [f for f in formats if f.get('is_from_start')]
2252
2253         def refetch_manifest(format_id, delay):
2254             nonlocal formats, start_time, is_live
2255             if time.time() <= start_time + delay:
2256                 return
2257
2258             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2259             video_details = traverse_obj(
2260                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2261             microformats = traverse_obj(
2262                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2263                 expected_type=dict, default=[])
2264             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2265             start_time = time.time()
2266
2267         def mpd_feed(format_id, delay):
2268             """
2269             @returns (manifest_url, manifest_stream_number, is_live) or None
2270             """
2271             with lock:
2272                 refetch_manifest(format_id, delay)
2273
2274             f = next((f for f in formats if f['format_id'] == format_id), None)
2275             if not f:
2276                 if not is_live:
2277                     self.to_screen(f'{video_id}: Video is no longer live')
2278                 else:
2279                     self.report_warning(
2280                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2281                 return None
2282             return f['manifest_url'], f['manifest_stream_number'], is_live
2283
2284         for f in formats:
2285             f['is_live'] = True
2286             f['protocol'] = 'http_dash_segments_generator'
2287             f['fragments'] = functools.partial(
2288                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2289
2290     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2291         FETCH_SPAN, MAX_DURATION = 5, 432000
2292
2293         mpd_url, stream_number, is_live = None, None, True
2294
2295         begin_index = 0
2296         download_start_time = ctx.get('start') or time.time()
2297
2298         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2299         if lack_early_segments:
2300             self.report_warning(bug_reports_message(
2301                 'Starting download from the last 120 hours of the live stream since '
2302                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2303             lack_early_segments = True
2304
2305         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2306         fragments, fragment_base_url = None, None
2307
2308         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2309             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2310             # Obtain from MPD's maximum seq value
2311             old_mpd_url = mpd_url
2312             last_error = ctx.pop('last_error', None)
2313             expire_fast = immediate or last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2314             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2315                                                or (mpd_url, stream_number, False))
2316             if not refresh_sequence:
2317                 if expire_fast and not is_live:
2318                     return False, last_seq
2319                 elif old_mpd_url == mpd_url:
2320                     return True, last_seq
2321             try:
2322                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2323                     mpd_url, None, note=False, errnote=False, fatal=False)
2324             except ExtractorError:
2325                 fmts = None
2326             if not fmts:
2327                 no_fragment_score += 2
2328                 return False, last_seq
2329             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2330             fragments = fmt_info['fragments']
2331             fragment_base_url = fmt_info['fragment_base_url']
2332             assert fragment_base_url
2333
2334             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2335             return True, _last_seq
2336
2337         while is_live:
2338             fetch_time = time.time()
2339             if no_fragment_score > 30:
2340                 return
2341             if last_segment_url:
2342                 # Obtain from "X-Head-Seqnum" header value from each segment
2343                 try:
2344                     urlh = self._request_webpage(
2345                         last_segment_url, None, note=False, errnote=False, fatal=False)
2346                 except ExtractorError:
2347                     urlh = None
2348                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2349                 if last_seq is None:
2350                     no_fragment_score += 2
2351                     last_segment_url = None
2352                     continue
2353             else:
2354                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2355                 no_fragment_score += 2
2356                 if not should_continue:
2357                     continue
2358
2359             if known_idx > last_seq:
2360                 last_segment_url = None
2361                 continue
2362
2363             last_seq += 1
2364
2365             if begin_index < 0 and known_idx < 0:
2366                 # skip from the start when it's negative value
2367                 known_idx = last_seq + begin_index
2368             if lack_early_segments:
2369                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2370             try:
2371                 for idx in range(known_idx, last_seq):
2372                     # do not update sequence here or you'll get skipped some part of it
2373                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2374                     if not should_continue:
2375                         known_idx = idx - 1
2376                         raise ExtractorError('breaking out of outer loop')
2377                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2378                     yield {
2379                         'url': last_segment_url,
2380                     }
2381                 if known_idx == last_seq:
2382                     no_fragment_score += 5
2383                 else:
2384                     no_fragment_score = 0
2385                 known_idx = last_seq
2386             except ExtractorError:
2387                 continue
2388
2389             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2390
2391     def _extract_player_url(self, *ytcfgs, webpage=None):
2392         player_url = traverse_obj(
2393             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2394             get_all=False, expected_type=compat_str)
2395         if not player_url:
2396             return
2397         return urljoin('https://www.youtube.com', player_url)
2398
2399     def _download_player_url(self, video_id, fatal=False):
2400         res = self._download_webpage(
2401             'https://www.youtube.com/iframe_api',
2402             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2403         if res:
2404             player_version = self._search_regex(
2405                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2406             if player_version:
2407                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2408
2409     def _signature_cache_id(self, example_sig):
2410         """ Return a string representation of a signature """
2411         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2412
2413     @classmethod
2414     def _extract_player_info(cls, player_url):
2415         for player_re in cls._PLAYER_INFO_RE:
2416             id_m = re.search(player_re, player_url)
2417             if id_m:
2418                 break
2419         else:
2420             raise ExtractorError('Cannot identify player %r' % player_url)
2421         return id_m.group('id')
2422
2423     def _load_player(self, video_id, player_url, fatal=True):
2424         player_id = self._extract_player_info(player_url)
2425         if player_id not in self._code_cache:
2426             code = self._download_webpage(
2427                 player_url, video_id, fatal=fatal,
2428                 note='Downloading player ' + player_id,
2429                 errnote='Download of %s failed' % player_url)
2430             if code:
2431                 self._code_cache[player_id] = code
2432         return self._code_cache.get(player_id)
2433
2434     def _extract_signature_function(self, video_id, player_url, example_sig):
2435         player_id = self._extract_player_info(player_url)
2436
2437         # Read from filesystem cache
2438         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2439         assert os.path.basename(func_id) == func_id
2440
2441         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
2442         if cache_spec is not None:
2443             return lambda s: ''.join(s[i] for i in cache_spec)
2444
2445         code = self._load_player(video_id, player_url)
2446         if code:
2447             res = self._parse_sig_js(code)
2448
2449             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2450             cache_res = res(test_string)
2451             cache_spec = [ord(c) for c in cache_res]
2452
2453             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
2454             return res
2455
2456     def _print_sig_code(self, func, example_sig):
2457         if not self.get_param('youtube_print_sig_code'):
2458             return
2459
2460         def gen_sig_code(idxs):
2461             def _genslice(start, end, step):
2462                 starts = '' if start == 0 else str(start)
2463                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2464                 steps = '' if step == 1 else (':%d' % step)
2465                 return f's[{starts}{ends}{steps}]'
2466
2467             step = None
2468             # Quelch pyflakes warnings - start will be set when step is set
2469             start = '(Never used)'
2470             for i, prev in zip(idxs[1:], idxs[:-1]):
2471                 if step is not None:
2472                     if i - prev == step:
2473                         continue
2474                     yield _genslice(start, prev, step)
2475                     step = None
2476                     continue
2477                 if i - prev in [-1, 1]:
2478                     step = i - prev
2479                     start = prev
2480                     continue
2481                 else:
2482                     yield 's[%d]' % prev
2483             if step is None:
2484                 yield 's[%d]' % i
2485             else:
2486                 yield _genslice(start, i, step)
2487
2488         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2489         cache_res = func(test_string)
2490         cache_spec = [ord(c) for c in cache_res]
2491         expr_code = ' + '.join(gen_sig_code(cache_spec))
2492         signature_id_tuple = '(%s)' % (
2493             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2494         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2495                 '    return %s\n') % (signature_id_tuple, expr_code)
2496         self.to_screen('Extracted signature function:\n' + code)
2497
2498     def _parse_sig_js(self, jscode):
2499         funcname = self._search_regex(
2500             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2501              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2502              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2503              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2504              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2505              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2506              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2507              # Obsolete patterns
2508              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2509              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2510              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2511              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2512              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2513              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2514              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2515              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2516             jscode, 'Initial JS player signature function name', group='sig')
2517
2518         jsi = JSInterpreter(jscode)
2519         initial_function = jsi.extract_function(funcname)
2520         return lambda s: initial_function([s])
2521
2522     def _decrypt_signature(self, s, video_id, player_url):
2523         """Turn the encrypted s field into a working signature"""
2524
2525         if player_url is None:
2526             raise ExtractorError('Cannot decrypt signature without player_url')
2527
2528         try:
2529             player_id = (player_url, self._signature_cache_id(s))
2530             if player_id not in self._player_cache:
2531                 func = self._extract_signature_function(
2532                     video_id, player_url, s
2533                 )
2534                 self._player_cache[player_id] = func
2535             func = self._player_cache[player_id]
2536             self._print_sig_code(func, s)
2537             return func(s)
2538         except Exception as e:
2539             raise ExtractorError('Signature extraction failed: ' + traceback.format_exc(), cause=e)
2540
2541     def _decrypt_nsig(self, s, video_id, player_url):
2542         """Turn the encrypted n field into a working signature"""
2543         if player_url is None:
2544             raise ExtractorError('Cannot decrypt nsig without player_url')
2545         player_url = urljoin('https://www.youtube.com', player_url)
2546
2547         sig_id = ('nsig_value', s)
2548         if sig_id in self._player_cache:
2549             return self._player_cache[sig_id]
2550
2551         try:
2552             player_id = ('nsig', player_url)
2553             if player_id not in self._player_cache:
2554                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2555             func = self._player_cache[player_id]
2556             self._player_cache[sig_id] = func(s)
2557             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2558             return self._player_cache[sig_id]
2559         except Exception as e:
2560             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2561
2562     def _extract_n_function_name(self, jscode):
2563         nfunc, idx = self._search_regex(
2564             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2565             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2566         if not idx:
2567             return nfunc
2568         return json.loads(js_to_json(self._search_regex(
2569             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2570             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2571
2572     def _extract_n_function(self, video_id, player_url):
2573         player_id = self._extract_player_info(player_url)
2574         func_code = self._downloader.cache.load('youtube-nsig', player_id)
2575
2576         if func_code:
2577             jsi = JSInterpreter(func_code)
2578         else:
2579             jscode = self._load_player(video_id, player_url)
2580             funcname = self._extract_n_function_name(jscode)
2581             jsi = JSInterpreter(jscode)
2582             func_code = jsi.extract_function_code(funcname)
2583             self._downloader.cache.store('youtube-nsig', player_id, func_code)
2584
2585         if self.get_param('youtube_print_sig_code'):
2586             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2587
2588         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2589
2590     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2591         """
2592         Extract signatureTimestamp (sts)
2593         Required to tell API what sig/player version is in use.
2594         """
2595         sts = None
2596         if isinstance(ytcfg, dict):
2597             sts = int_or_none(ytcfg.get('STS'))
2598
2599         if not sts:
2600             # Attempt to extract from player
2601             if player_url is None:
2602                 error_msg = 'Cannot extract signature timestamp without player_url.'
2603                 if fatal:
2604                     raise ExtractorError(error_msg)
2605                 self.report_warning(error_msg)
2606                 return
2607             code = self._load_player(video_id, player_url, fatal=fatal)
2608             if code:
2609                 sts = int_or_none(self._search_regex(
2610                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2611                     'JS player signature timestamp', group='sts', fatal=fatal))
2612         return sts
2613
2614     def _mark_watched(self, video_id, player_responses):
2615         playback_url = get_first(
2616             player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2617             expected_type=url_or_none)
2618         if not playback_url:
2619             self.report_warning('Unable to mark watched')
2620             return
2621         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2622         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2623
2624         # cpn generation algorithm is reverse engineered from base.js.
2625         # In fact it works even with dummy cpn.
2626         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2627         cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2628
2629         qs.update({
2630             'ver': ['2'],
2631             'cpn': [cpn],
2632         })
2633         playback_url = compat_urlparse.urlunparse(
2634             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2635
2636         self._download_webpage(
2637             playback_url, video_id, 'Marking watched',
2638             'Unable to mark watched', fatal=False)
2639
2640     @staticmethod
2641     def _extract_urls(webpage):
2642         # Embedded YouTube player
2643         entries = [
2644             unescapeHTML(mobj.group('url'))
2645             for mobj in re.finditer(r'''(?x)
2646             (?:
2647                 <iframe[^>]+?src=|
2648                 data-video-url=|
2649                 <embed[^>]+?src=|
2650                 embedSWF\(?:\s*|
2651                 <object[^>]+data=|
2652                 new\s+SWFObject\(
2653             )
2654             (["\'])
2655                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2656                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2657             \1''', webpage)]
2658
2659         # lazyYT YouTube embed
2660         entries.extend(list(map(
2661             unescapeHTML,
2662             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2663
2664         # Wordpress "YouTube Video Importer" plugin
2665         matches = re.findall(r'''(?x)<div[^>]+
2666             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2667             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2668         entries.extend(m[-1] for m in matches)
2669
2670         return entries
2671
2672     @staticmethod
2673     def _extract_url(webpage):
2674         urls = YoutubeIE._extract_urls(webpage)
2675         return urls[0] if urls else None
2676
2677     @classmethod
2678     def extract_id(cls, url):
2679         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2680         if mobj is None:
2681             raise ExtractorError('Invalid URL: %s' % url)
2682         return mobj.group('id')
2683
2684     def _extract_chapters_from_json(self, data, duration):
2685         chapter_list = traverse_obj(
2686             data, (
2687                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2688                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2689             ), expected_type=list)
2690
2691         return self._extract_chapters(
2692             chapter_list,
2693             chapter_time=lambda chapter: float_or_none(
2694                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2695             chapter_title=lambda chapter: traverse_obj(
2696                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2697             duration=duration)
2698
2699     def _extract_chapters_from_engagement_panel(self, data, duration):
2700         content_list = traverse_obj(
2701             data,
2702             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2703             expected_type=list, default=[])
2704         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2705         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2706
2707         return next((
2708             filter(None, (
2709                 self._extract_chapters(
2710                     traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2711                     chapter_time, chapter_title, duration)
2712                 for contents in content_list
2713             ))), [])
2714
2715     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration):
2716         chapters = []
2717         last_chapter = {'start_time': 0}
2718         for idx, chapter in enumerate(chapter_list or []):
2719             title = chapter_title(chapter)
2720             start_time = chapter_time(chapter)
2721             if start_time is None:
2722                 continue
2723             last_chapter['end_time'] = start_time
2724             if start_time < last_chapter['start_time']:
2725                 if idx == 1:
2726                     chapters.pop()
2727                     self.report_warning('Invalid start time for chapter "%s"' % last_chapter['title'])
2728                 else:
2729                     self.report_warning(f'Invalid start time for chapter "{title}"')
2730                     continue
2731             last_chapter = {'start_time': start_time, 'title': title}
2732             chapters.append(last_chapter)
2733         last_chapter['end_time'] = duration
2734         return chapters
2735
2736     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
2737         return self._parse_json(self._search_regex(
2738             (fr'{regex}\s*{self._YT_INITIAL_BOUNDARY_RE}',
2739              regex), webpage, name, default='{}'), video_id, fatal=False)
2740
2741     def _extract_comment(self, comment_renderer, parent=None):
2742         comment_id = comment_renderer.get('commentId')
2743         if not comment_id:
2744             return
2745
2746         text = self._get_text(comment_renderer, 'contentText')
2747
2748         # note: timestamp is an estimate calculated from the current time and time_text
2749         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2750         author = self._get_text(comment_renderer, 'authorText')
2751         author_id = try_get(comment_renderer,
2752                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2753
2754         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2755                                                        lambda x: x['likeCount']), compat_str)) or 0
2756         author_thumbnail = try_get(comment_renderer,
2757                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2758
2759         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2760         is_favorited = 'creatorHeart' in (try_get(
2761             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2762         return {
2763             'id': comment_id,
2764             'text': text,
2765             'timestamp': timestamp,
2766             'time_text': time_text,
2767             'like_count': votes,
2768             'is_favorited': is_favorited,
2769             'author': author,
2770             'author_id': author_id,
2771             'author_thumbnail': author_thumbnail,
2772             'author_is_uploader': author_is_uploader,
2773             'parent': parent or 'root'
2774         }
2775
2776     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2777
2778         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2779
2780         def extract_header(contents):
2781             _continuation = None
2782             for content in contents:
2783                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2784                 expected_comment_count = self._get_count(
2785                     comments_header_renderer, 'countText', 'commentsCount')
2786
2787                 if expected_comment_count:
2788                     tracker['est_total'] = expected_comment_count
2789                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2790                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2791
2792                 sort_menu_item = try_get(
2793                     comments_header_renderer,
2794                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2795                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2796
2797                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2798                 if not _continuation:
2799                     continue
2800
2801                 sort_text = str_or_none(sort_menu_item.get('title'))
2802                 if not sort_text:
2803                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2804                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2805                 break
2806             return _continuation
2807
2808         def extract_thread(contents):
2809             if not parent:
2810                 tracker['current_page_thread'] = 0
2811             for content in contents:
2812                 if not parent and tracker['total_parent_comments'] >= max_parents:
2813                     yield
2814                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2815                 comment_renderer = get_first(
2816                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2817                     expected_type=dict, default={})
2818
2819                 comment = self._extract_comment(comment_renderer, parent)
2820                 if not comment:
2821                     continue
2822
2823                 tracker['running_total'] += 1
2824                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2825                 yield comment
2826
2827                 # Attempt to get the replies
2828                 comment_replies_renderer = try_get(
2829                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2830
2831                 if comment_replies_renderer:
2832                     tracker['current_page_thread'] += 1
2833                     comment_entries_iter = self._comment_entries(
2834                         comment_replies_renderer, ytcfg, video_id,
2835                         parent=comment.get('id'), tracker=tracker)
2836                     yield from itertools.islice(comment_entries_iter, min(
2837                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2838
2839         # Keeps track of counts across recursive calls
2840         if not tracker:
2841             tracker = dict(
2842                 running_total=0,
2843                 est_total=0,
2844                 current_page_thread=0,
2845                 total_parent_comments=0,
2846                 total_reply_comments=0)
2847
2848         # TODO: Deprecated
2849         # YouTube comments have a max depth of 2
2850         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2851         if max_depth:
2852             self._downloader.deprecation_warning(
2853                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2854         if max_depth == 1 and parent:
2855             return
2856
2857         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2858             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2859
2860         continuation = self._extract_continuation(root_continuation_data)
2861
2862         response = None
2863         is_forced_continuation = False
2864         is_first_continuation = parent is None
2865         if is_first_continuation and not continuation:
2866             # Sometimes you can get comments by generating the continuation yourself,
2867             # even if YouTube initially reports them being disabled - e.g. stories comments.
2868             # Note: if the comment section is actually disabled, YouTube may return a response with
2869             # required check_get_keys missing. So we will disable that check initially in this case.
2870             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2871             is_forced_continuation = True
2872
2873         for page_num in itertools.count(0):
2874             if not continuation:
2875                 break
2876             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2877             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2878             if page_num == 0:
2879                 if is_first_continuation:
2880                     note_prefix = 'Downloading comment section API JSON'
2881                 else:
2882                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2883                         tracker['current_page_thread'], comment_prog_str)
2884             else:
2885                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2886                     '       ' if parent else '', ' replies' if parent else '',
2887                     page_num, comment_prog_str)
2888
2889             response = self._extract_response(
2890                 item_id=None, query=continuation,
2891                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2892                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2893             is_forced_continuation = False
2894             continuation_contents = traverse_obj(
2895                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2896
2897             continuation = None
2898             for continuation_section in continuation_contents:
2899                 continuation_items = traverse_obj(
2900                     continuation_section,
2901                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2902                     get_all=False, expected_type=list) or []
2903                 if is_first_continuation:
2904                     continuation = extract_header(continuation_items)
2905                     is_first_continuation = False
2906                     if continuation:
2907                         break
2908                     continue
2909
2910                 for entry in extract_thread(continuation_items):
2911                     if not entry:
2912                         return
2913                     yield entry
2914                 continuation = self._extract_continuation({'contents': continuation_items})
2915                 if continuation:
2916                     break
2917
2918         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2919         if message and not parent and tracker['running_total'] == 0:
2920             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2921
2922     @staticmethod
2923     def _generate_comment_continuation(video_id):
2924         """
2925         Generates initial comment section continuation token from given video id
2926         """
2927         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2928         return base64.b64encode(token.encode()).decode()
2929
2930     def _get_comments(self, ytcfg, video_id, contents, webpage):
2931         """Entry for comment extraction"""
2932         def _real_comment_extract(contents):
2933             renderer = next((
2934                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2935                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2936             yield from self._comment_entries(renderer, ytcfg, video_id)
2937
2938         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2939         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2940
2941     @staticmethod
2942     def _get_checkok_params():
2943         return {'contentCheckOk': True, 'racyCheckOk': True}
2944
2945     @classmethod
2946     def _generate_player_context(cls, sts=None):
2947         context = {
2948             'html5Preference': 'HTML5_PREF_WANTS',
2949         }
2950         if sts is not None:
2951             context['signatureTimestamp'] = sts
2952         return {
2953             'playbackContext': {
2954                 'contentPlaybackContext': context
2955             },
2956             **cls._get_checkok_params()
2957         }
2958
2959     @staticmethod
2960     def _is_agegated(player_response):
2961         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2962             return True
2963
2964         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2965         AGE_GATE_REASONS = (
2966             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2967             'age_verification_required', 'age_check_required',  # status
2968         )
2969         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
2970
2971     @staticmethod
2972     def _is_unplayable(player_response):
2973         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
2974
2975     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
2976
2977         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
2978         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
2979         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
2980         headers = self.generate_api_headers(
2981             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
2982
2983         yt_query = {
2984             'videoId': video_id,
2985             'params': '8AEB'  # enable stories
2986         }
2987         yt_query.update(self._generate_player_context(sts))
2988         return self._extract_response(
2989             item_id=video_id, ep='player', query=yt_query,
2990             ytcfg=player_ytcfg, headers=headers, fatal=True,
2991             default_client=client,
2992             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
2993         ) or None
2994
2995     def _get_requested_clients(self, url, smuggled_data):
2996         requested_clients = []
2997         default = ['android', 'web']
2998         allowed_clients = sorted(
2999             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3000             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3001         for client in self._configuration_arg('player_client'):
3002             if client in allowed_clients:
3003                 requested_clients.append(client)
3004             elif client == 'default':
3005                 requested_clients.extend(default)
3006             elif client == 'all':
3007                 requested_clients.extend(allowed_clients)
3008             else:
3009                 self.report_warning(f'Skipping unsupported client {client}')
3010         if not requested_clients:
3011             requested_clients = default
3012
3013         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3014             requested_clients.extend(
3015                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3016
3017         return orderedSet(requested_clients)
3018
3019     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3020         initial_pr = None
3021         if webpage:
3022             initial_pr = self._extract_yt_initial_variable(
3023                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
3024                 video_id, 'initial player response')
3025
3026         all_clients = set(clients)
3027         clients = clients[::-1]
3028         prs = []
3029
3030         def append_client(*client_names):
3031             """ Append the first client name that exists but not already used """
3032             for client_name in client_names:
3033                 actual_client = _split_innertube_client(client_name)[0]
3034                 if actual_client in INNERTUBE_CLIENTS:
3035                     if actual_client not in all_clients:
3036                         clients.append(client_name)
3037                         all_clients.add(actual_client)
3038                         return
3039
3040         # Android player_response does not have microFormats which are needed for
3041         # extraction of some data. So we return the initial_pr with formats
3042         # stripped out even if not requested by the user
3043         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3044         if initial_pr:
3045             pr = dict(initial_pr)
3046             pr['streamingData'] = None
3047             prs.append(pr)
3048
3049         last_error = None
3050         tried_iframe_fallback = False
3051         player_url = None
3052         while clients:
3053             client, base_client, variant = _split_innertube_client(clients.pop())
3054             player_ytcfg = master_ytcfg if client == 'web' else {}
3055             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3056                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3057
3058             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3059             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3060             if 'js' in self._configuration_arg('player_skip'):
3061                 require_js_player = False
3062                 player_url = None
3063
3064             if not player_url and not tried_iframe_fallback and require_js_player:
3065                 player_url = self._download_player_url(video_id)
3066                 tried_iframe_fallback = True
3067
3068             try:
3069                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3070                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3071             except ExtractorError as e:
3072                 if last_error:
3073                     self.report_warning(last_error)
3074                 last_error = e
3075                 continue
3076
3077             if pr:
3078                 prs.append(pr)
3079
3080             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3081             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3082                 append_client(f'{base_client}_creator')
3083             elif self._is_agegated(pr):
3084                 if variant == 'tv_embedded':
3085                     append_client(f'{base_client}_embedded')
3086                 elif not variant:
3087                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3088
3089         if last_error:
3090             if not len(prs):
3091                 raise last_error
3092             self.report_warning(last_error)
3093         return prs, player_url
3094
3095     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3096         itags, stream_ids = {}, []
3097         itag_qualities, res_qualities = {}, {}
3098         q = qualities([
3099             # Normally tiny is the smallest video-only formats. But
3100             # audio-only formats with unknown quality may get tagged as tiny
3101             'tiny',
3102             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3103             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3104         ])
3105         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3106
3107         for fmt in streaming_formats:
3108             if fmt.get('targetDurationSec'):
3109                 continue
3110
3111             itag = str_or_none(fmt.get('itag'))
3112             audio_track = fmt.get('audioTrack') or {}
3113             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3114             if stream_id in stream_ids:
3115                 continue
3116
3117             quality = fmt.get('quality')
3118             height = int_or_none(fmt.get('height'))
3119             if quality == 'tiny' or not quality:
3120                 quality = fmt.get('audioQuality', '').lower() or quality
3121             # The 3gp format (17) in android client has a quality of "small",
3122             # but is actually worse than other formats
3123             if itag == '17':
3124                 quality = 'tiny'
3125             if quality:
3126                 if itag:
3127                     itag_qualities[itag] = quality
3128                 if height:
3129                     res_qualities[height] = quality
3130             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3131             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3132             # number of fragment that would subsequently requested with (`&sq=N`)
3133             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3134                 continue
3135
3136             fmt_url = fmt.get('url')
3137             if not fmt_url:
3138                 sc = compat_parse_qs(fmt.get('signatureCipher'))
3139                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3140                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3141                 if not (sc and fmt_url and encrypted_sig):
3142                     continue
3143                 if not player_url:
3144                     continue
3145                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
3146                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
3147                 fmt_url += '&' + sp + '=' + signature
3148
3149             query = parse_qs(fmt_url)
3150             throttled = False
3151             if query.get('n'):
3152                 try:
3153                     fmt_url = update_url_query(fmt_url, {
3154                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3155                 except ExtractorError as e:
3156                     self.report_warning(
3157                         'nsig extraction failed: You may experience throttling for some formats\n'
3158                         f'n = {query["n"][0]} ; player = {player_url}\n{e}', only_once=True)
3159                     throttled = True
3160
3161             if itag:
3162                 itags[itag] = 'https'
3163                 stream_ids.append(stream_id)
3164
3165             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3166             language_preference = (
3167                 10 if audio_track.get('audioIsDefault') and 10
3168                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3169                 else -1)
3170             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3171             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3172             # Make sure to avoid false positives with small duration differences.
3173             # Eg: __2ABJjxzNo, ySuUZEjARPY
3174             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3175             if is_damaged:
3176                 self.report_warning(f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3177             dct = {
3178                 'asr': int_or_none(fmt.get('audioSampleRate')),
3179                 'filesize': int_or_none(fmt.get('contentLength')),
3180                 'format_id': itag,
3181                 'format_note': join_nonempty(
3182                     '%s%s' % (audio_track.get('displayName') or '',
3183                               ' (default)' if language_preference > 0 else ''),
3184                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3185                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3186                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3187                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3188                 'fps': int_or_none(fmt.get('fps')) or None,
3189                 'height': height,
3190                 'quality': q(quality),
3191                 'has_drm': bool(fmt.get('drmFamilies')),
3192                 'tbr': tbr,
3193                 'url': fmt_url,
3194                 'width': int_or_none(fmt.get('width')),
3195                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3196                                           'desc' if language_preference < -1 else ''),
3197                 'language_preference': language_preference,
3198                 # Strictly de-prioritize damaged and 3gp formats
3199                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3200             }
3201             mime_mobj = re.match(
3202                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3203             if mime_mobj:
3204                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3205                 dct.update(parse_codecs(mime_mobj.group(2)))
3206             no_audio = dct.get('acodec') == 'none'
3207             no_video = dct.get('vcodec') == 'none'
3208             if no_audio:
3209                 dct['vbr'] = tbr
3210             if no_video:
3211                 dct['abr'] = tbr
3212             if no_audio or no_video:
3213                 dct['downloader_options'] = {
3214                     # Youtube throttles chunks >~10M
3215                     'http_chunk_size': 10485760,
3216                 }
3217                 if dct.get('ext'):
3218                     dct['container'] = dct['ext'] + '_dash'
3219             yield dct
3220
3221         live_from_start = is_live and self.get_param('live_from_start')
3222         skip_manifests = self._configuration_arg('skip')
3223         if not self.get_param('youtube_include_hls_manifest', True):
3224             skip_manifests.append('hls')
3225         get_dash = 'dash' not in skip_manifests and (
3226             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3227         get_hls = not live_from_start and 'hls' not in skip_manifests
3228
3229         def process_manifest_format(f, proto, itag):
3230             if itag in itags:
3231                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3232                     return False
3233                 itag = f'{itag}-{proto}'
3234             if itag:
3235                 f['format_id'] = itag
3236                 itags[itag] = proto
3237
3238             f['quality'] = next((
3239                 q(qdict[val])
3240                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3241                 if val in qdict), -1)
3242             return True
3243
3244         for sd in streaming_data:
3245             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3246             if hls_manifest_url:
3247                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3248                     if process_manifest_format(f, 'hls', self._search_regex(
3249                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3250                         yield f
3251
3252             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3253             if dash_manifest_url:
3254                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3255                     if process_manifest_format(f, 'dash', f['format_id']):
3256                         f['filesize'] = int_or_none(self._search_regex(
3257                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3258                         if live_from_start:
3259                             f['is_from_start'] = True
3260
3261                         yield f
3262
3263     def _extract_storyboard(self, player_responses, duration):
3264         spec = get_first(
3265             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3266         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3267         if not base_url:
3268             return
3269         L = len(spec) - 1
3270         for i, args in enumerate(spec):
3271             args = args.split('#')
3272             counts = list(map(int_or_none, args[:5]))
3273             if len(args) != 8 or not all(counts):
3274                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3275                 continue
3276             width, height, frame_count, cols, rows = counts
3277             N, sigh = args[6:]
3278
3279             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3280             fragment_count = frame_count / (cols * rows)
3281             fragment_duration = duration / fragment_count
3282             yield {
3283                 'format_id': f'sb{i}',
3284                 'format_note': 'storyboard',
3285                 'ext': 'mhtml',
3286                 'protocol': 'mhtml',
3287                 'acodec': 'none',
3288                 'vcodec': 'none',
3289                 'url': url,
3290                 'width': width,
3291                 'height': height,
3292                 'fragments': [{
3293                     'url': url.replace('$M', str(j)),
3294                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3295                 } for j in range(math.ceil(fragment_count))],
3296             }
3297
3298     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3299         webpage = None
3300         if 'webpage' not in self._configuration_arg('player_skip'):
3301             webpage = self._download_webpage(
3302                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3303
3304         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3305
3306         player_responses, player_url = self._extract_player_responses(
3307             self._get_requested_clients(url, smuggled_data),
3308             video_id, webpage, master_ytcfg)
3309
3310         return webpage, master_ytcfg, player_responses, player_url
3311
3312     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3313         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3314         is_live = get_first(video_details, 'isLive')
3315         if is_live is None:
3316             is_live = get_first(live_broadcast_details, 'isLiveNow')
3317
3318         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3319         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3320
3321         return live_broadcast_details, is_live, streaming_data, formats
3322
3323     def _real_extract(self, url):
3324         url, smuggled_data = unsmuggle_url(url, {})
3325         video_id = self._match_id(url)
3326
3327         base_url = self.http_scheme() + '//www.youtube.com/'
3328         webpage_url = base_url + 'watch?v=' + video_id
3329
3330         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3331
3332         playability_statuses = traverse_obj(
3333             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3334
3335         trailer_video_id = get_first(
3336             playability_statuses,
3337             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3338             expected_type=str)
3339         if trailer_video_id:
3340             return self.url_result(
3341                 trailer_video_id, self.ie_key(), trailer_video_id)
3342
3343         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3344                        if webpage else (lambda x: None))
3345
3346         video_details = traverse_obj(
3347             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3348         microformats = traverse_obj(
3349             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3350             expected_type=dict, default=[])
3351         video_title = (
3352             get_first(video_details, 'title')
3353             or self._get_text(microformats, (..., 'title'))
3354             or search_meta(['og:title', 'twitter:title', 'title']))
3355         video_description = get_first(video_details, 'shortDescription')
3356
3357         multifeed_metadata_list = get_first(
3358             player_responses,
3359             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3360             expected_type=str)
3361         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3362             if self.get_param('noplaylist'):
3363                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3364             else:
3365                 entries = []
3366                 feed_ids = []
3367                 for feed in multifeed_metadata_list.split(','):
3368                     # Unquote should take place before split on comma (,) since textual
3369                     # fields may contain comma as well (see
3370                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3371                     feed_data = compat_parse_qs(
3372                         compat_urllib_parse_unquote_plus(feed))
3373
3374                     def feed_entry(name):
3375                         return try_get(
3376                             feed_data, lambda x: x[name][0], compat_str)
3377
3378                     feed_id = feed_entry('id')
3379                     if not feed_id:
3380                         continue
3381                     feed_title = feed_entry('title')
3382                     title = video_title
3383                     if feed_title:
3384                         title += ' (%s)' % feed_title
3385                     entries.append({
3386                         '_type': 'url_transparent',
3387                         'ie_key': 'Youtube',
3388                         'url': smuggle_url(
3389                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3390                             {'force_singlefeed': True}),
3391                         'title': title,
3392                     })
3393                     feed_ids.append(feed_id)
3394                 self.to_screen(
3395                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3396                     % (', '.join(feed_ids), video_id))
3397                 return self.playlist_result(
3398                     entries, video_id, video_title, video_description)
3399
3400         duration = int_or_none(
3401             get_first(video_details, 'lengthSeconds')
3402             or get_first(microformats, 'lengthSeconds')
3403             or parse_duration(search_meta('duration'))) or None
3404
3405         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3406             video_id, microformats, video_details, player_responses, player_url, duration)
3407
3408         if not formats:
3409             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3410                 self.report_drm(video_id)
3411             pemr = get_first(
3412                 playability_statuses,
3413                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3414             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3415             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3416             if subreason:
3417                 if subreason == 'The uploader has not made this video available in your country.':
3418                     countries = get_first(microformats, 'availableCountries')
3419                     if not countries:
3420                         regions_allowed = search_meta('regionsAllowed')
3421                         countries = regions_allowed.split(',') if regions_allowed else None
3422                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3423                 reason += f'. {subreason}'
3424             if reason:
3425                 self.raise_no_formats(reason, expected=True)
3426
3427         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3428         if not keywords and webpage:
3429             keywords = [
3430                 unescapeHTML(m.group('content'))
3431                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3432         for keyword in keywords:
3433             if keyword.startswith('yt:stretch='):
3434                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3435                 if mobj:
3436                     # NB: float is intentional for forcing float division
3437                     w, h = (float(v) for v in mobj.groups())
3438                     if w > 0 and h > 0:
3439                         ratio = w / h
3440                         for f in formats:
3441                             if f.get('vcodec') != 'none':
3442                                 f['stretched_ratio'] = ratio
3443                         break
3444         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3445         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3446         if thumbnail_url:
3447             thumbnails.append({
3448                 'url': thumbnail_url,
3449             })
3450         original_thumbnails = thumbnails.copy()
3451
3452         # The best resolution thumbnails sometimes does not appear in the webpage
3453         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3454         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3455         thumbnail_names = [
3456             # While the *1,*2,*3 thumbnails are just below their correspnding "*default" variants
3457             # in resolution, these are not the custom thumbnail. So de-prioritize them
3458             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3459             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3460         ]
3461         n_thumbnail_names = len(thumbnail_names)
3462         thumbnails.extend({
3463             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3464                 video_id=video_id, name=name, ext=ext,
3465                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3466         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3467         for thumb in thumbnails:
3468             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3469             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3470         self._remove_duplicate_formats(thumbnails)
3471         self._downloader._sort_thumbnails(original_thumbnails)
3472
3473         category = get_first(microformats, 'category') or search_meta('genre')
3474         channel_id = str_or_none(
3475             get_first(video_details, 'channelId')
3476             or get_first(microformats, 'externalChannelId')
3477             or search_meta('channelId'))
3478         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3479
3480         live_content = get_first(video_details, 'isLiveContent')
3481         is_upcoming = get_first(video_details, 'isUpcoming')
3482         if is_live is None:
3483             if is_upcoming or live_content is False:
3484                 is_live = False
3485         if is_upcoming is None and (live_content or is_live):
3486             is_upcoming = False
3487         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3488         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3489         if not duration and live_end_time and live_start_time:
3490             duration = live_end_time - live_start_time
3491
3492         if is_live and self.get_param('live_from_start'):
3493             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3494
3495         formats.extend(self._extract_storyboard(player_responses, duration))
3496
3497         # Source is given priority since formats that throttle are given lower source_preference
3498         # When throttling issue is fully fixed, remove this
3499         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3500
3501         info = {
3502             'id': video_id,
3503             'title': video_title,
3504             'formats': formats,
3505             'thumbnails': thumbnails,
3506             # The best thumbnail that we are sure exists. Prevents unnecessary
3507             # URL checking if user don't care about getting the best possible thumbnail
3508             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3509             'description': video_description,
3510             'uploader': get_first(video_details, 'author'),
3511             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3512             'uploader_url': owner_profile_url,
3513             'channel_id': channel_id,
3514             'channel_url': format_field(channel_id, template='https://www.youtube.com/channel/%s'),
3515             'duration': duration,
3516             'view_count': int_or_none(
3517                 get_first((video_details, microformats), (..., 'viewCount'))
3518                 or search_meta('interactionCount')),
3519             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3520             'age_limit': 18 if (
3521                 get_first(microformats, 'isFamilySafe') is False
3522                 or search_meta('isFamilyFriendly') == 'false'
3523                 or search_meta('og:restrictions:age') == '18+') else 0,
3524             'webpage_url': webpage_url,
3525             'categories': [category] if category else None,
3526             'tags': keywords,
3527             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3528             'is_live': is_live,
3529             'was_live': (False if is_live or is_upcoming or live_content is False
3530                          else None if is_live is None or is_upcoming is None
3531                          else live_content),
3532             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3533             'release_timestamp': live_start_time,
3534         }
3535
3536         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3537         if pctr:
3538             def get_lang_code(track):
3539                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3540                         or track.get('languageCode'))
3541
3542             # Converted into dicts to remove duplicates
3543             captions = {
3544                 get_lang_code(sub): sub
3545                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3546             translation_languages = {
3547                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3548                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3549
3550             def process_language(container, base_url, lang_code, sub_name, query):
3551                 lang_subs = container.setdefault(lang_code, [])
3552                 for fmt in self._SUBTITLE_FORMATS:
3553                     query.update({
3554                         'fmt': fmt,
3555                     })
3556                     lang_subs.append({
3557                         'ext': fmt,
3558                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3559                         'name': sub_name,
3560                     })
3561
3562             subtitles, automatic_captions = {}, {}
3563             for lang_code, caption_track in captions.items():
3564                 base_url = caption_track.get('baseUrl')
3565                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3566                 if not base_url:
3567                     continue
3568                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3569                 if caption_track.get('kind') != 'asr':
3570                     if not lang_code:
3571                         continue
3572                     process_language(
3573                         subtitles, base_url, lang_code, lang_name, {})
3574                     if not caption_track.get('isTranslatable'):
3575                         continue
3576                 for trans_code, trans_name in translation_languages.items():
3577                     if not trans_code:
3578                         continue
3579                     orig_trans_code = trans_code
3580                     if caption_track.get('kind') != 'asr':
3581                         if 'translated_subs' in self._configuration_arg('skip'):
3582                             continue
3583                         trans_code += f'-{lang_code}'
3584                         trans_name += format_field(lang_name, template=' from %s')
3585                     # Add an "-orig" label to the original language so that it can be distinguished.
3586                     # The subs are returned without "-orig" as well for compatibility
3587                     if lang_code == f'a-{orig_trans_code}':
3588                         process_language(
3589                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3590                     # Setting tlang=lang returns damaged subtitles.
3591                     process_language(automatic_captions, base_url, trans_code, trans_name,
3592                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3593             info['automatic_captions'] = automatic_captions
3594             info['subtitles'] = subtitles
3595
3596         parsed_url = compat_urllib_parse_urlparse(url)
3597         for component in [parsed_url.fragment, parsed_url.query]:
3598             query = compat_parse_qs(component)
3599             for k, v in query.items():
3600                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3601                     d_k += '_time'
3602                     if d_k not in info and k in s_ks:
3603                         info[d_k] = parse_duration(query[k][0])
3604
3605         # Youtube Music Auto-generated description
3606         if video_description:
3607             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
3608             if mobj:
3609                 release_year = mobj.group('release_year')
3610                 release_date = mobj.group('release_date')
3611                 if release_date:
3612                     release_date = release_date.replace('-', '')
3613                     if not release_year:
3614                         release_year = release_date[:4]
3615                 info.update({
3616                     'album': mobj.group('album'.strip()),
3617                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3618                     'track': mobj.group('track').strip(),
3619                     'release_date': release_date,
3620                     'release_year': int_or_none(release_year),
3621                 })
3622
3623         initial_data = None
3624         if webpage:
3625             initial_data = self._extract_yt_initial_variable(
3626                 webpage, self._YT_INITIAL_DATA_RE, video_id,
3627                 'yt initial data')
3628         if not initial_data:
3629             query = {'videoId': video_id}
3630             query.update(self._get_checkok_params())
3631             initial_data = self._extract_response(
3632                 item_id=video_id, ep='next', fatal=False,
3633                 ytcfg=master_ytcfg, query=query,
3634                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3635                 note='Downloading initial data API JSON')
3636
3637         try:  # This will error if there is no livechat
3638             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3639         except (KeyError, IndexError, TypeError):
3640             pass
3641         else:
3642             info.setdefault('subtitles', {})['live_chat'] = [{
3643                 'url': f'https://www.youtube.com/watch?v={video_id}',  # url is needed to set cookies
3644                 'video_id': video_id,
3645                 'ext': 'json',
3646                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3647             }]
3648
3649         if initial_data:
3650             info['chapters'] = (
3651                 self._extract_chapters_from_json(initial_data, duration)
3652                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3653                 or None)
3654
3655         contents = traverse_obj(
3656             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3657             expected_type=list, default=[])
3658
3659         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3660         if vpir:
3661             stl = vpir.get('superTitleLink')
3662             if stl:
3663                 stl = self._get_text(stl)
3664                 if try_get(
3665                         vpir,
3666                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3667                     info['location'] = stl
3668                 else:
3669                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3670                     if mobj:
3671                         info.update({
3672                             'series': mobj.group(1),
3673                             'season_number': int(mobj.group(2)),
3674                             'episode_number': int(mobj.group(3)),
3675                         })
3676             for tlb in (try_get(
3677                     vpir,
3678                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3679                     list) or []):
3680                 tbr = tlb.get('toggleButtonRenderer') or {}
3681                 for getter, regex in [(
3682                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3683                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3684                             lambda x: x['accessibility'],
3685                             lambda x: x['accessibilityData']['accessibilityData'],
3686                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3687                     label = (try_get(tbr, getter, dict) or {}).get('label')
3688                     if label:
3689                         mobj = re.match(regex, label)
3690                         if mobj:
3691                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3692                             break
3693             sbr_tooltip = try_get(
3694                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3695             if sbr_tooltip:
3696                 like_count, dislike_count = sbr_tooltip.split(' / ')
3697                 info.update({
3698                     'like_count': str_to_int(like_count),
3699                     'dislike_count': str_to_int(dislike_count),
3700                 })
3701         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3702         if vsir:
3703             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3704             info.update({
3705                 'channel': self._get_text(vor, 'title'),
3706                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3707
3708             rows = try_get(
3709                 vsir,
3710                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3711                 list) or []
3712             multiple_songs = False
3713             for row in rows:
3714                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3715                     multiple_songs = True
3716                     break
3717             for row in rows:
3718                 mrr = row.get('metadataRowRenderer') or {}
3719                 mrr_title = mrr.get('title')
3720                 if not mrr_title:
3721                     continue
3722                 mrr_title = self._get_text(mrr, 'title')
3723                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3724                 if mrr_title == 'License':
3725                     info['license'] = mrr_contents_text
3726                 elif not multiple_songs:
3727                     if mrr_title == 'Album':
3728                         info['album'] = mrr_contents_text
3729                     elif mrr_title == 'Artist':
3730                         info['artist'] = mrr_contents_text
3731                     elif mrr_title == 'Song':
3732                         info['track'] = mrr_contents_text
3733
3734         fallbacks = {
3735             'channel': 'uploader',
3736             'channel_id': 'uploader_id',
3737             'channel_url': 'uploader_url',
3738         }
3739
3740         # The upload date for scheduled, live and past live streams / premieres in microformats
3741         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3742         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3743         upload_date = (
3744             unified_strdate(get_first(microformats, 'uploadDate'))
3745             or unified_strdate(search_meta('uploadDate')))
3746         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3747             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3748         info['upload_date'] = upload_date
3749
3750         for to, frm in fallbacks.items():
3751             if not info.get(to):
3752                 info[to] = info.get(frm)
3753
3754         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3755             v = info.get(s_k)
3756             if v:
3757                 info[d_k] = v
3758
3759         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3760         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3761         is_membersonly = None
3762         is_premium = None
3763         if initial_data and is_private is not None:
3764             is_membersonly = False
3765             is_premium = False
3766             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3767             badge_labels = set()
3768             for content in contents:
3769                 if not isinstance(content, dict):
3770                     continue
3771                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3772             for badge_label in badge_labels:
3773                 if badge_label.lower() == 'members only':
3774                     is_membersonly = True
3775                 elif badge_label.lower() == 'premium':
3776                     is_premium = True
3777                 elif badge_label.lower() == 'unlisted':
3778                     is_unlisted = True
3779
3780         info['availability'] = self._availability(
3781             is_private=is_private,
3782             needs_premium=is_premium,
3783             needs_subscription=is_membersonly,
3784             needs_auth=info['age_limit'] >= 18,
3785             is_unlisted=None if is_private is None else is_unlisted)
3786
3787         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3788
3789         self.mark_watched(video_id, player_responses)
3790
3791         return info
3792
3793
3794 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3795
3796     @staticmethod
3797     def passthrough_smuggled_data(func):
3798         def _smuggle(entries, smuggled_data):
3799             for entry in entries:
3800                 # TODO: Convert URL to music.youtube instead.
3801                 # Do we need to passthrough any other smuggled_data?
3802                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3803                 yield entry
3804
3805         @functools.wraps(func)
3806         def wrapper(self, url):
3807             url, smuggled_data = unsmuggle_url(url, {})
3808             if self.is_music_url(url):
3809                 smuggled_data['is_music_url'] = True
3810             info_dict = func(self, url, smuggled_data)
3811             if smuggled_data and info_dict.get('entries'):
3812                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3813             return info_dict
3814         return wrapper
3815
3816     def _extract_channel_id(self, webpage):
3817         channel_id = self._html_search_meta(
3818             'channelId', webpage, 'channel id', default=None)
3819         if channel_id:
3820             return channel_id
3821         channel_url = self._html_search_meta(
3822             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3823              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3824              'twitter:app:url:googleplay'), webpage, 'channel url')
3825         return self._search_regex(
3826             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3827             channel_url, 'channel id')
3828
3829     @staticmethod
3830     def _extract_basic_item_renderer(item):
3831         # Modified from _extract_grid_item_renderer
3832         known_basic_renderers = (
3833             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3834         )
3835         for key, renderer in item.items():
3836             if not isinstance(renderer, dict):
3837                 continue
3838             elif key in known_basic_renderers:
3839                 return renderer
3840             elif key.startswith('grid') and key.endswith('Renderer'):
3841                 return renderer
3842
3843     def _grid_entries(self, grid_renderer):
3844         for item in grid_renderer['items']:
3845             if not isinstance(item, dict):
3846                 continue
3847             renderer = self._extract_basic_item_renderer(item)
3848             if not isinstance(renderer, dict):
3849                 continue
3850             title = self._get_text(renderer, 'title')
3851
3852             # playlist
3853             playlist_id = renderer.get('playlistId')
3854             if playlist_id:
3855                 yield self.url_result(
3856                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3857                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3858                     video_title=title)
3859                 continue
3860             # video
3861             video_id = renderer.get('videoId')
3862             if video_id:
3863                 yield self._extract_video(renderer)
3864                 continue
3865             # channel
3866             channel_id = renderer.get('channelId')
3867             if channel_id:
3868                 yield self.url_result(
3869                     'https://www.youtube.com/channel/%s' % channel_id,
3870                     ie=YoutubeTabIE.ie_key(), video_title=title)
3871                 continue
3872             # generic endpoint URL support
3873             ep_url = urljoin('https://www.youtube.com/', try_get(
3874                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3875                 compat_str))
3876             if ep_url:
3877                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3878                     if ie.suitable(ep_url):
3879                         yield self.url_result(
3880                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3881                         break
3882
3883     def _music_reponsive_list_entry(self, renderer):
3884         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3885         if video_id:
3886             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3887                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3888         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3889         if playlist_id:
3890             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3891             if video_id:
3892                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3893                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3894             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3895                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3896         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3897         if browse_id:
3898             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3899                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3900
3901     def _shelf_entries_from_content(self, shelf_renderer):
3902         content = shelf_renderer.get('content')
3903         if not isinstance(content, dict):
3904             return
3905         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3906         if renderer:
3907             # TODO: add support for nested playlists so each shelf is processed
3908             # as separate playlist
3909             # TODO: this includes only first N items
3910             yield from self._grid_entries(renderer)
3911         renderer = content.get('horizontalListRenderer')
3912         if renderer:
3913             # TODO
3914             pass
3915
3916     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3917         ep = try_get(
3918             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3919             compat_str)
3920         shelf_url = urljoin('https://www.youtube.com', ep)
3921         if shelf_url:
3922             # Skipping links to another channels, note that checking for
3923             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3924             # will not work
3925             if skip_channels and '/channels?' in shelf_url:
3926                 return
3927             title = self._get_text(shelf_renderer, 'title')
3928             yield self.url_result(shelf_url, video_title=title)
3929         # Shelf may not contain shelf URL, fallback to extraction from content
3930         yield from self._shelf_entries_from_content(shelf_renderer)
3931
3932     def _playlist_entries(self, video_list_renderer):
3933         for content in video_list_renderer['contents']:
3934             if not isinstance(content, dict):
3935                 continue
3936             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3937             if not isinstance(renderer, dict):
3938                 continue
3939             video_id = renderer.get('videoId')
3940             if not video_id:
3941                 continue
3942             yield self._extract_video(renderer)
3943
3944     def _rich_entries(self, rich_grid_renderer):
3945         renderer = try_get(
3946             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3947         video_id = renderer.get('videoId')
3948         if not video_id:
3949             return
3950         yield self._extract_video(renderer)
3951
3952     def _video_entry(self, video_renderer):
3953         video_id = video_renderer.get('videoId')
3954         if video_id:
3955             return self._extract_video(video_renderer)
3956
3957     def _hashtag_tile_entry(self, hashtag_tile_renderer):
3958         url = urljoin('https://youtube.com', traverse_obj(
3959             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
3960         if url:
3961             return self.url_result(
3962                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
3963
3964     def _post_thread_entries(self, post_thread_renderer):
3965         post_renderer = try_get(
3966             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
3967         if not post_renderer:
3968             return
3969         # video attachment
3970         video_renderer = try_get(
3971             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
3972         video_id = video_renderer.get('videoId')
3973         if video_id:
3974             entry = self._extract_video(video_renderer)
3975             if entry:
3976                 yield entry
3977         # playlist attachment
3978         playlist_id = try_get(
3979             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
3980         if playlist_id:
3981             yield self.url_result(
3982                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
3983                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3984         # inline video links
3985         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
3986         for run in runs:
3987             if not isinstance(run, dict):
3988                 continue
3989             ep_url = try_get(
3990                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
3991             if not ep_url:
3992                 continue
3993             if not YoutubeIE.suitable(ep_url):
3994                 continue
3995             ep_video_id = YoutubeIE._match_id(ep_url)
3996             if video_id == ep_video_id:
3997                 continue
3998             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
3999
4000     def _post_thread_continuation_entries(self, post_thread_continuation):
4001         contents = post_thread_continuation.get('contents')
4002         if not isinstance(contents, list):
4003             return
4004         for content in contents:
4005             renderer = content.get('backstagePostThreadRenderer')
4006             if not isinstance(renderer, dict):
4007                 continue
4008             yield from self._post_thread_entries(renderer)
4009
4010     r''' # unused
4011     def _rich_grid_entries(self, contents):
4012         for content in contents:
4013             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4014             if video_renderer:
4015                 entry = self._video_entry(video_renderer)
4016                 if entry:
4017                     yield entry
4018     '''
4019
4020     def _extract_entries(self, parent_renderer, continuation_list):
4021         # continuation_list is modified in-place with continuation_list = [continuation_token]
4022         continuation_list[:] = [None]
4023         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4024         for content in contents:
4025             if not isinstance(content, dict):
4026                 continue
4027             is_renderer = traverse_obj(
4028                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4029                 expected_type=dict)
4030             if not is_renderer:
4031                 renderer = content.get('richItemRenderer')
4032                 if renderer:
4033                     for entry in self._rich_entries(renderer):
4034                         yield entry
4035                     continuation_list[0] = self._extract_continuation(parent_renderer)
4036                 continue
4037             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4038             for isr_content in isr_contents:
4039                 if not isinstance(isr_content, dict):
4040                     continue
4041
4042                 known_renderers = {
4043                     'playlistVideoListRenderer': self._playlist_entries,
4044                     'gridRenderer': self._grid_entries,
4045                     'reelShelfRenderer': self._grid_entries,
4046                     'shelfRenderer': self._shelf_entries,
4047                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4048                     'backstagePostThreadRenderer': self._post_thread_entries,
4049                     'videoRenderer': lambda x: [self._video_entry(x)],
4050                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4051                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4052                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4053                 }
4054                 for key, renderer in isr_content.items():
4055                     if key not in known_renderers:
4056                         continue
4057                     for entry in known_renderers[key](renderer):
4058                         if entry:
4059                             yield entry
4060                     continuation_list[0] = self._extract_continuation(renderer)
4061                     break
4062
4063             if not continuation_list[0]:
4064                 continuation_list[0] = self._extract_continuation(is_renderer)
4065
4066         if not continuation_list[0]:
4067             continuation_list[0] = self._extract_continuation(parent_renderer)
4068
4069     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4070         continuation_list = [None]
4071         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4072         tab_content = try_get(tab, lambda x: x['content'], dict)
4073         if not tab_content:
4074             return
4075         parent_renderer = (
4076             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4077             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4078         yield from extract_entries(parent_renderer)
4079         continuation = continuation_list[0]
4080
4081         for page_num in itertools.count(1):
4082             if not continuation:
4083                 break
4084             headers = self.generate_api_headers(
4085                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4086             response = self._extract_response(
4087                 item_id=f'{item_id} page {page_num}',
4088                 query=continuation, headers=headers, ytcfg=ytcfg,
4089                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4090
4091             if not response:
4092                 break
4093             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4094             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4095             visitor_data = self._extract_visitor_data(response) or visitor_data
4096
4097             known_continuation_renderers = {
4098                 'playlistVideoListContinuation': self._playlist_entries,
4099                 'gridContinuation': self._grid_entries,
4100                 'itemSectionContinuation': self._post_thread_continuation_entries,
4101                 'sectionListContinuation': extract_entries,  # for feeds
4102             }
4103             continuation_contents = try_get(
4104                 response, lambda x: x['continuationContents'], dict) or {}
4105             continuation_renderer = None
4106             for key, value in continuation_contents.items():
4107                 if key not in known_continuation_renderers:
4108                     continue
4109                 continuation_renderer = value
4110                 continuation_list = [None]
4111                 yield from known_continuation_renderers[key](continuation_renderer)
4112                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4113                 break
4114             if continuation_renderer:
4115                 continue
4116
4117             known_renderers = {
4118                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4119                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4120                 'gridVideoRenderer': (self._grid_entries, 'items'),
4121                 'gridChannelRenderer': (self._grid_entries, 'items'),
4122                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4123                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4124                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4125                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4126             }
4127             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4128             continuation_items = try_get(
4129                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4130             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4131             video_items_renderer = None
4132             for key, value in continuation_item.items():
4133                 if key not in known_renderers:
4134                     continue
4135                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4136                 continuation_list = [None]
4137                 yield from known_renderers[key][0](video_items_renderer)
4138                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4139                 break
4140             if video_items_renderer:
4141                 continue
4142             break
4143
4144     @staticmethod
4145     def _extract_selected_tab(tabs, fatal=True):
4146         for tab in tabs:
4147             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4148             if renderer.get('selected') is True:
4149                 return renderer
4150         else:
4151             if fatal:
4152                 raise ExtractorError('Unable to find selected tab')
4153
4154     def _extract_uploader(self, data):
4155         uploader = {}
4156         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4157         owner = try_get(
4158             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4159         if owner:
4160             owner_text = owner.get('text')
4161             uploader['uploader'] = self._search_regex(
4162                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4163             uploader['uploader_id'] = try_get(
4164                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
4165             uploader['uploader_url'] = urljoin(
4166                 'https://www.youtube.com/',
4167                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
4168         return {k: v for k, v in uploader.items() if v is not None}
4169
4170     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4171         playlist_id = title = description = channel_url = channel_name = channel_id = None
4172         tags = []
4173
4174         selected_tab = self._extract_selected_tab(tabs)
4175         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4176         renderer = try_get(
4177             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4178         if renderer:
4179             channel_name = renderer.get('title')
4180             channel_url = renderer.get('channelUrl')
4181             channel_id = renderer.get('externalId')
4182         else:
4183             renderer = try_get(
4184                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4185
4186         if renderer:
4187             title = renderer.get('title')
4188             description = renderer.get('description', '')
4189             playlist_id = channel_id
4190             tags = renderer.get('keywords', '').split()
4191
4192         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4193         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4194         def _get_uncropped(url):
4195             return url_or_none((url or '').split('=')[0] + '=s0')
4196
4197         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4198         if avatar_thumbnails:
4199             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4200             if uncropped_avatar:
4201                 avatar_thumbnails.append({
4202                     'url': uncropped_avatar,
4203                     'id': 'avatar_uncropped',
4204                     'preference': 1
4205                 })
4206
4207         channel_banners = self._extract_thumbnails(
4208             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4209         for banner in channel_banners:
4210             banner['preference'] = -10
4211
4212         if channel_banners:
4213             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4214             if uncropped_banner:
4215                 channel_banners.append({
4216                     'url': uncropped_banner,
4217                     'id': 'banner_uncropped',
4218                     'preference': -5
4219                 })
4220
4221         primary_thumbnails = self._extract_thumbnails(
4222             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4223
4224         if playlist_id is None:
4225             playlist_id = item_id
4226
4227         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4228         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4229         if title is None:
4230             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4231         title += format_field(selected_tab, 'title', ' - %s')
4232         title += format_field(selected_tab, 'expandedText', ' - %s')
4233
4234         metadata = {
4235             'playlist_id': playlist_id,
4236             'playlist_title': title,
4237             'playlist_description': description,
4238             'uploader': channel_name,
4239             'uploader_id': channel_id,
4240             'uploader_url': channel_url,
4241             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4242             'tags': tags,
4243             'view_count': self._get_count(playlist_stats, 1),
4244             'availability': self._extract_availability(data),
4245             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4246             'playlist_count': self._get_count(playlist_stats, 0),
4247             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4248         }
4249         if not channel_id:
4250             metadata.update(self._extract_uploader(data))
4251         metadata.update({
4252             'channel': metadata['uploader'],
4253             'channel_id': metadata['uploader_id'],
4254             'channel_url': metadata['uploader_url']})
4255         return self.playlist_result(
4256             self._entries(
4257                 selected_tab, playlist_id, ytcfg,
4258                 self._extract_account_syncid(ytcfg, data),
4259                 self._extract_visitor_data(data, ytcfg)),
4260             **metadata)
4261
4262     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4263         first_id = last_id = response = None
4264         for page_num in itertools.count(1):
4265             videos = list(self._playlist_entries(playlist))
4266             if not videos:
4267                 return
4268             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4269             if start >= len(videos):
4270                 return
4271             for video in videos[start:]:
4272                 yield video
4273             first_id = first_id or videos[0]['id']
4274             last_id = videos[-1]['id']
4275             watch_endpoint = try_get(
4276                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4277             headers = self.generate_api_headers(
4278                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4279                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4280             query = {
4281                 'playlistId': playlist_id,
4282                 'videoId': watch_endpoint.get('videoId') or last_id,
4283                 'index': watch_endpoint.get('index') or len(videos),
4284                 'params': watch_endpoint.get('params') or 'OAE%3D'
4285             }
4286             response = self._extract_response(
4287                 item_id='%s page %d' % (playlist_id, page_num),
4288                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4289                 check_get_keys='contents'
4290             )
4291             playlist = try_get(
4292                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4293
4294     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4295         title = playlist.get('title') or try_get(
4296             data, lambda x: x['titleText']['simpleText'], compat_str)
4297         playlist_id = playlist.get('playlistId') or item_id
4298
4299         # Delegating everything except mix playlists to regular tab-based playlist URL
4300         playlist_url = urljoin(url, try_get(
4301             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4302             compat_str))
4303
4304         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4305         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4306         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4307
4308         if playlist_url and playlist_url != url and not is_known_unviewable:
4309             return self.url_result(
4310                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4311                 video_title=title)
4312
4313         return self.playlist_result(
4314             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4315             playlist_id=playlist_id, playlist_title=title)
4316
4317     def _extract_availability(self, data):
4318         """
4319         Gets the availability of a given playlist/tab.
4320         Note: Unless YouTube tells us explicitly, we do not assume it is public
4321         @param data: response
4322         """
4323         is_private = is_unlisted = None
4324         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4325         badge_labels = self._extract_badges(renderer)
4326
4327         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4328         privacy_dropdown_entries = try_get(
4329             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4330         for renderer_dict in privacy_dropdown_entries:
4331             is_selected = try_get(
4332                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4333             if not is_selected:
4334                 continue
4335             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4336             if label:
4337                 badge_labels.add(label.lower())
4338                 break
4339
4340         for badge_label in badge_labels:
4341             if badge_label == 'unlisted':
4342                 is_unlisted = True
4343             elif badge_label == 'private':
4344                 is_private = True
4345             elif badge_label == 'public':
4346                 is_unlisted = is_private = False
4347         return self._availability(is_private, False, False, False, is_unlisted)
4348
4349     @staticmethod
4350     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4351         sidebar_renderer = try_get(
4352             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4353         for item in sidebar_renderer:
4354             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4355             if renderer:
4356                 return renderer
4357
4358     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4359         """
4360         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4361         """
4362         browse_id = params = None
4363         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4364         if not renderer:
4365             return
4366         menu_renderer = try_get(
4367             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4368         for menu_item in menu_renderer:
4369             if not isinstance(menu_item, dict):
4370                 continue
4371             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4372             text = try_get(
4373                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4374             if not text or text.lower() != 'show unavailable videos':
4375                 continue
4376             browse_endpoint = try_get(
4377                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4378             browse_id = browse_endpoint.get('browseId')
4379             params = browse_endpoint.get('params')
4380             break
4381
4382         headers = self.generate_api_headers(
4383             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4384             visitor_data=self._extract_visitor_data(data, ytcfg))
4385         query = {
4386             'params': params or 'wgYCCAA=',
4387             'browseId': browse_id or 'VL%s' % item_id
4388         }
4389         return self._extract_response(
4390             item_id=item_id, headers=headers, query=query,
4391             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4392             note='Downloading API JSON with unavailable videos')
4393
4394     @property
4395     def skip_webpage(self):
4396         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4397
4398     def _extract_webpage(self, url, item_id, fatal=True):
4399         retries = self.get_param('extractor_retries', 3)
4400         count = -1
4401         webpage = data = last_error = None
4402         while count < retries:
4403             count += 1
4404             # Sometimes youtube returns a webpage with incomplete ytInitialData
4405             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4406             if last_error:
4407                 self.report_warning('%s. Retrying ...' % last_error)
4408             try:
4409                 webpage = self._download_webpage(
4410                     url, item_id,
4411                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4412                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4413             except ExtractorError as e:
4414                 if isinstance(e.cause, network_exceptions):
4415                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4416                         last_error = error_to_compat_str(e.cause or e.msg)
4417                         if count < retries:
4418                             continue
4419                 if fatal:
4420                     raise
4421                 self.report_warning(error_to_compat_str(e))
4422                 break
4423             else:
4424                 try:
4425                     self._extract_and_report_alerts(data)
4426                 except ExtractorError as e:
4427                     if fatal:
4428                         raise
4429                     self.report_warning(error_to_compat_str(e))
4430                     break
4431
4432                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4433                     break
4434
4435                 last_error = 'Incomplete yt initial data received'
4436                 if count >= retries:
4437                     if fatal:
4438                         raise ExtractorError(last_error)
4439                     self.report_warning(last_error)
4440                     break
4441
4442         return webpage, data
4443
4444     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4445         """Use if failed to extract ytcfg (and data) from initial webpage"""
4446         if not ytcfg and self.is_authenticated:
4447             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4448             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4449                 raise ExtractorError(
4450                     f'{msg}. If you are not downloading private content, or '
4451                     'your cookies are only for the first account and channel,'
4452                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4453                     expected=True)
4454             self.report_warning(msg, only_once=True)
4455
4456     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4457         data = None
4458         if not self.skip_webpage:
4459             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4460             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4461             # Reject webpage data if redirected to home page without explicitly requesting
4462             selected_tab = self._extract_selected_tab(traverse_obj(
4463                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4464             if (url != 'https://www.youtube.com/feed/recommended'
4465                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4466                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4467                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4468                 if fatal:
4469                     raise ExtractorError(msg, expected=True)
4470                 self.report_warning(msg, only_once=True)
4471         if not data:
4472             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4473             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4474         return data, ytcfg
4475
4476     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4477         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4478         resolve_response = self._extract_response(
4479             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4480             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4481         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4482         for ep_key, ep in endpoints.items():
4483             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4484             if params:
4485                 return self._extract_response(
4486                     item_id=item_id, query=params, ep=ep, headers=headers,
4487                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4488                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4489         err_note = 'Failed to resolve url (does the playlist exist?)'
4490         if fatal:
4491             raise ExtractorError(err_note, expected=True)
4492         self.report_warning(err_note, item_id)
4493
4494     _SEARCH_PARAMS = None
4495
4496     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4497         data = {'query': query}
4498         if params is NO_DEFAULT:
4499             params = self._SEARCH_PARAMS
4500         if params:
4501             data['params'] = params
4502
4503         content_keys = (
4504             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4505             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4506             # ytmusic search
4507             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4508             ('continuationContents', ),
4509         )
4510         display_id = f'query "{query}"'
4511         check_get_keys = tuple({keys[0] for keys in content_keys})
4512         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4513         self._report_playlist_authcheck(ytcfg, fatal=False)
4514
4515         continuation_list = [None]
4516         search = None
4517         for page_num in itertools.count(1):
4518             data.update(continuation_list[0] or {})
4519             headers = self.generate_api_headers(
4520                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4521             search = self._extract_response(
4522                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4523                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4524             slr_contents = traverse_obj(search, *content_keys)
4525             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4526             if not continuation_list[0]:
4527                 break
4528
4529
4530 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4531     IE_DESC = 'YouTube Tabs'
4532     _VALID_URL = r'''(?x:
4533         https?://
4534             (?:\w+\.)?
4535             (?:
4536                 youtube(?:kids)?\.com|
4537                 %(invidious)s
4538             )/
4539             (?:
4540                 (?P<channel_type>channel|c|user|browse)/|
4541                 (?P<not_channel>
4542                     feed/|hashtag/|
4543                     (?:playlist|watch)\?.*?\blist=
4544                 )|
4545                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4546             )
4547             (?P<id>[^/?\#&]+)
4548     )''' % {
4549         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4550         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4551     }
4552     IE_NAME = 'youtube:tab'
4553
4554     _TESTS = [{
4555         'note': 'playlists, multipage',
4556         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4557         'playlist_mincount': 94,
4558         'info_dict': {
4559             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4560             'title': 'Igor Kleiner - Playlists',
4561             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4562             'uploader': 'Igor Kleiner',
4563             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4564             'channel': 'Igor Kleiner',
4565             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4566             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4567             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4568             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4569             'channel_follower_count': int
4570         },
4571     }, {
4572         'note': 'playlists, multipage, different order',
4573         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4574         'playlist_mincount': 94,
4575         'info_dict': {
4576             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4577             'title': 'Igor Kleiner - Playlists',
4578             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4579             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4580             'uploader': 'Igor Kleiner',
4581             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4582             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4583             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4584             'channel': 'Igor Kleiner',
4585             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4586             'channel_follower_count': int
4587         },
4588     }, {
4589         'note': 'playlists, series',
4590         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4591         'playlist_mincount': 5,
4592         'info_dict': {
4593             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4594             'title': '3Blue1Brown - Playlists',
4595             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4596             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4597             'uploader': '3Blue1Brown',
4598             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4599             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4600             'channel': '3Blue1Brown',
4601             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4602             'tags': ['Mathematics'],
4603             'channel_follower_count': int
4604         },
4605     }, {
4606         'note': 'playlists, singlepage',
4607         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4608         'playlist_mincount': 4,
4609         'info_dict': {
4610             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4611             'title': 'ThirstForScience - Playlists',
4612             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4613             'uploader': 'ThirstForScience',
4614             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4615             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4616             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4617             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4618             'tags': 'count:13',
4619             'channel': 'ThirstForScience',
4620             'channel_follower_count': int
4621         }
4622     }, {
4623         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4624         'only_matching': True,
4625     }, {
4626         'note': 'basic, single video playlist',
4627         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4628         'info_dict': {
4629             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4630             'uploader': 'Sergey M.',
4631             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4632             'title': 'youtube-dl public playlist',
4633             'description': '',
4634             'tags': [],
4635             'view_count': int,
4636             'modified_date': '20201130',
4637             'channel': 'Sergey M.',
4638             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4639             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4640             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4641         },
4642         'playlist_count': 1,
4643     }, {
4644         'note': 'empty playlist',
4645         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4646         'info_dict': {
4647             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4648             'uploader': 'Sergey M.',
4649             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4650             'title': 'youtube-dl empty playlist',
4651             'tags': [],
4652             'channel': 'Sergey M.',
4653             'description': '',
4654             'modified_date': '20160902',
4655             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4656             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4657             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4658         },
4659         'playlist_count': 0,
4660     }, {
4661         'note': 'Home tab',
4662         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4663         'info_dict': {
4664             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4665             'title': 'lex will - Home',
4666             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4667             'uploader': 'lex will',
4668             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4669             'channel': 'lex will',
4670             'tags': ['bible', 'history', 'prophesy'],
4671             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4672             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4673             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4674             'channel_follower_count': int
4675         },
4676         'playlist_mincount': 2,
4677     }, {
4678         'note': 'Videos tab',
4679         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4680         'info_dict': {
4681             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4682             'title': 'lex will - Videos',
4683             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4684             'uploader': 'lex will',
4685             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4686             'tags': ['bible', 'history', 'prophesy'],
4687             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4688             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4689             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4690             'channel': 'lex will',
4691             'channel_follower_count': int
4692         },
4693         'playlist_mincount': 975,
4694     }, {
4695         'note': 'Videos tab, sorted by popular',
4696         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4697         'info_dict': {
4698             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4699             'title': 'lex will - Videos',
4700             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4701             'uploader': 'lex will',
4702             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4703             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4704             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4705             'channel': 'lex will',
4706             'tags': ['bible', 'history', 'prophesy'],
4707             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4708             'channel_follower_count': int
4709         },
4710         'playlist_mincount': 199,
4711     }, {
4712         'note': 'Playlists tab',
4713         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4714         'info_dict': {
4715             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4716             'title': 'lex will - Playlists',
4717             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4718             'uploader': 'lex will',
4719             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4720             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4721             'channel': 'lex will',
4722             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4723             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4724             'tags': ['bible', 'history', 'prophesy'],
4725             'channel_follower_count': int
4726         },
4727         'playlist_mincount': 17,
4728     }, {
4729         'note': 'Community tab',
4730         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4731         'info_dict': {
4732             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4733             'title': 'lex will - Community',
4734             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4735             'uploader': 'lex will',
4736             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4737             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4738             'channel': 'lex will',
4739             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4740             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4741             'tags': ['bible', 'history', 'prophesy'],
4742             'channel_follower_count': int
4743         },
4744         'playlist_mincount': 18,
4745     }, {
4746         'note': 'Channels tab',
4747         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4748         'info_dict': {
4749             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4750             'title': 'lex will - Channels',
4751             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4752             'uploader': 'lex will',
4753             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4754             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4755             'channel': 'lex will',
4756             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4757             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4758             'tags': ['bible', 'history', 'prophesy'],
4759             'channel_follower_count': int
4760         },
4761         'playlist_mincount': 12,
4762     }, {
4763         'note': 'Search tab',
4764         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4765         'playlist_mincount': 40,
4766         'info_dict': {
4767             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4768             'title': '3Blue1Brown - Search - linear algebra',
4769             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4770             'uploader': '3Blue1Brown',
4771             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4772             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4773             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4774             'tags': ['Mathematics'],
4775             'channel': '3Blue1Brown',
4776             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4777             'channel_follower_count': int
4778         },
4779     }, {
4780         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4781         'only_matching': True,
4782     }, {
4783         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4784         'only_matching': True,
4785     }, {
4786         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4787         'only_matching': True,
4788     }, {
4789         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4790         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4791         'info_dict': {
4792             'title': '29C3: Not my department',
4793             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4794             'uploader': 'Christiaan008',
4795             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4796             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4797             'tags': [],
4798             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4799             'view_count': int,
4800             'modified_date': '20150605',
4801             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4802             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4803             'channel': 'Christiaan008',
4804         },
4805         'playlist_count': 96,
4806     }, {
4807         'note': 'Large playlist',
4808         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4809         'info_dict': {
4810             'title': 'Uploads from Cauchemar',
4811             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4812             'uploader': 'Cauchemar',
4813             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4814             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4815             'tags': [],
4816             'modified_date': r're:\d{8}',
4817             'channel': 'Cauchemar',
4818             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4819             'view_count': int,
4820             'description': '',
4821             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4822         },
4823         'playlist_mincount': 1123,
4824         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4825     }, {
4826         'note': 'even larger playlist, 8832 videos',
4827         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4828         'only_matching': True,
4829     }, {
4830         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4831         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4832         'info_dict': {
4833             'title': 'Uploads from Interstellar Movie',
4834             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4835             'uploader': 'Interstellar Movie',
4836             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4837             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4838             'tags': [],
4839             'view_count': int,
4840             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4841             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4842             'channel': 'Interstellar Movie',
4843             'description': '',
4844             'modified_date': r're:\d{8}',
4845         },
4846         'playlist_mincount': 21,
4847     }, {
4848         'note': 'Playlist with "show unavailable videos" button',
4849         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4850         'info_dict': {
4851             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4852             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4853             'uploader': 'Phim Siêu Nhân Nhật Bản',
4854             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4855             'view_count': int,
4856             'channel': 'Phim Siêu Nhân Nhật Bản',
4857             'tags': [],
4858             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4859             'description': '',
4860             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4861             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4862             'modified_date': r're:\d{8}',
4863         },
4864         'playlist_mincount': 200,
4865         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4866     }, {
4867         'note': 'Playlist with unavailable videos in page 7',
4868         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4869         'info_dict': {
4870             'title': 'Uploads from BlankTV',
4871             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4872             'uploader': 'BlankTV',
4873             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4874             'channel': 'BlankTV',
4875             'channel_url': 'https://www.youtube.com/c/blanktv',
4876             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4877             'view_count': int,
4878             'tags': [],
4879             'uploader_url': 'https://www.youtube.com/c/blanktv',
4880             'modified_date': r're:\d{8}',
4881             'description': '',
4882         },
4883         'playlist_mincount': 1000,
4884         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4885     }, {
4886         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4887         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4888         'info_dict': {
4889             'title': 'Data Analysis with Dr Mike Pound',
4890             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4891             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4892             'uploader': 'Computerphile',
4893             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4894             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4895             'tags': [],
4896             'view_count': int,
4897             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4898             'channel_url': 'https://www.youtube.com/user/Computerphile',
4899             'channel': 'Computerphile',
4900         },
4901         'playlist_mincount': 11,
4902     }, {
4903         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4904         'only_matching': True,
4905     }, {
4906         'note': 'Playlist URL that does not actually serve a playlist',
4907         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4908         'info_dict': {
4909             'id': 'FqZTN594JQw',
4910             'ext': 'webm',
4911             'title': "Smiley's People 01 detective, Adventure Series, Action",
4912             'uploader': 'STREEM',
4913             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4914             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4915             'upload_date': '20150526',
4916             'license': 'Standard YouTube License',
4917             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4918             'categories': ['People & Blogs'],
4919             'tags': list,
4920             'view_count': int,
4921             'like_count': int,
4922         },
4923         'params': {
4924             'skip_download': True,
4925         },
4926         'skip': 'This video is not available.',
4927         'add_ie': [YoutubeIE.ie_key()],
4928     }, {
4929         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
4930         'only_matching': True,
4931     }, {
4932         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
4933         'only_matching': True,
4934     }, {
4935         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
4936         'info_dict': {
4937             'id': 'GgL890LIznQ',  # This will keep changing
4938             'ext': 'mp4',
4939             'title': str,
4940             'uploader': 'Sky News',
4941             'uploader_id': 'skynews',
4942             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
4943             'upload_date': r're:\d{8}',
4944             'description': str,
4945             'categories': ['News & Politics'],
4946             'tags': list,
4947             'like_count': int,
4948             'release_timestamp': 1642502819,
4949             'channel': 'Sky News',
4950             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
4951             'age_limit': 0,
4952             'view_count': int,
4953             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
4954             'playable_in_embed': True,
4955             'release_date': '20220118',
4956             'availability': 'public',
4957             'live_status': 'is_live',
4958             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
4959             'channel_follower_count': int
4960         },
4961         'params': {
4962             'skip_download': True,
4963         },
4964         'expected_warnings': ['Ignoring subtitle tracks found in '],
4965     }, {
4966         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
4967         'info_dict': {
4968             'id': 'a48o2S1cPoo',
4969             'ext': 'mp4',
4970             'title': 'The Young Turks - Live Main Show',
4971             'uploader': 'The Young Turks',
4972             'uploader_id': 'TheYoungTurks',
4973             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
4974             'upload_date': '20150715',
4975             'license': 'Standard YouTube License',
4976             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
4977             'categories': ['News & Politics'],
4978             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
4979             'like_count': int,
4980         },
4981         'params': {
4982             'skip_download': True,
4983         },
4984         'only_matching': True,
4985     }, {
4986         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
4987         'only_matching': True,
4988     }, {
4989         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
4990         'only_matching': True,
4991     }, {
4992         'note': 'A channel that is not live. Should raise error',
4993         'url': 'https://www.youtube.com/user/numberphile/live',
4994         'only_matching': True,
4995     }, {
4996         'url': 'https://www.youtube.com/feed/trending',
4997         'only_matching': True,
4998     }, {
4999         'url': 'https://www.youtube.com/feed/library',
5000         'only_matching': True,
5001     }, {
5002         'url': 'https://www.youtube.com/feed/history',
5003         'only_matching': True,
5004     }, {
5005         'url': 'https://www.youtube.com/feed/subscriptions',
5006         'only_matching': True,
5007     }, {
5008         'url': 'https://www.youtube.com/feed/watch_later',
5009         'only_matching': True,
5010     }, {
5011         'note': 'Recommended - redirects to home page.',
5012         'url': 'https://www.youtube.com/feed/recommended',
5013         'only_matching': True,
5014     }, {
5015         'note': 'inline playlist with not always working continuations',
5016         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5017         'only_matching': True,
5018     }, {
5019         'url': 'https://www.youtube.com/course',
5020         'only_matching': True,
5021     }, {
5022         'url': 'https://www.youtube.com/zsecurity',
5023         'only_matching': True,
5024     }, {
5025         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5026         'only_matching': True,
5027     }, {
5028         'url': 'https://www.youtube.com/TheYoungTurks/live',
5029         'only_matching': True,
5030     }, {
5031         'url': 'https://www.youtube.com/hashtag/cctv9',
5032         'info_dict': {
5033             'id': 'cctv9',
5034             'title': '#cctv9',
5035             'tags': [],
5036         },
5037         'playlist_mincount': 350,
5038     }, {
5039         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5040         'only_matching': True,
5041     }, {
5042         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5043         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5044         'only_matching': True
5045     }, {
5046         'note': '/browse/ should redirect to /channel/',
5047         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5048         'only_matching': True
5049     }, {
5050         'note': 'VLPL, should redirect to playlist?list=PL...',
5051         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5052         'info_dict': {
5053             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5054             'uploader': 'NoCopyrightSounds',
5055             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5056             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5057             'title': 'NCS Releases',
5058             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5059             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5060             'modified_date': r're:\d{8}',
5061             'view_count': int,
5062             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5063             'tags': [],
5064             'channel': 'NoCopyrightSounds',
5065         },
5066         'playlist_mincount': 166,
5067         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5068     }, {
5069         'note': 'Topic, should redirect to playlist?list=UU...',
5070         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5071         'info_dict': {
5072             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5073             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5074             'title': 'Uploads from Royalty Free Music - Topic',
5075             'uploader': 'Royalty Free Music - Topic',
5076             'tags': [],
5077             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5078             'channel': 'Royalty Free Music - Topic',
5079             'view_count': int,
5080             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5081             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5082             'modified_date': r're:\d{8}',
5083             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5084             'description': '',
5085         },
5086         'expected_warnings': [
5087             'The URL does not have a videos tab',
5088             r'[Uu]navailable videos (are|will be) hidden',
5089         ],
5090         'playlist_mincount': 101,
5091     }, {
5092         'note': 'Topic without a UU playlist',
5093         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5094         'info_dict': {
5095             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5096             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5097             'tags': [],
5098         },
5099         'expected_warnings': [
5100             'the playlist redirect gave error',
5101         ],
5102         'playlist_mincount': 9,
5103     }, {
5104         'note': 'Youtube music Album',
5105         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5106         'info_dict': {
5107             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5108             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5109             'tags': [],
5110             'view_count': int,
5111             'description': '',
5112             'availability': 'unlisted',
5113             'modified_date': r're:\d{8}',
5114         },
5115         'playlist_count': 50,
5116     }, {
5117         'note': 'unlisted single video playlist',
5118         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5119         'info_dict': {
5120             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5121             'uploader': 'colethedj',
5122             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5123             'title': 'yt-dlp unlisted playlist test',
5124             'availability': 'unlisted',
5125             'tags': [],
5126             'modified_date': '20211208',
5127             'channel': 'colethedj',
5128             'view_count': int,
5129             'description': '',
5130             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5131             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5132             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5133         },
5134         'playlist_count': 1,
5135     }, {
5136         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5137         'url': 'https://www.youtube.com/feed/recommended',
5138         'info_dict': {
5139             'id': 'recommended',
5140             'title': 'recommended',
5141             'tags': [],
5142         },
5143         'playlist_mincount': 50,
5144         'params': {
5145             'skip_download': True,
5146             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5147         },
5148     }, {
5149         'note': 'API Fallback: /videos tab, sorted by oldest first',
5150         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5151         'info_dict': {
5152             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5153             'title': 'Cody\'sLab - Videos',
5154             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5155             'uploader': 'Cody\'sLab',
5156             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5157             'channel': 'Cody\'sLab',
5158             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5159             'tags': [],
5160             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5161             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5162             'channel_follower_count': int
5163         },
5164         'playlist_mincount': 650,
5165         'params': {
5166             'skip_download': True,
5167             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5168         },
5169     }, {
5170         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5171         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5172         'info_dict': {
5173             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5174             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5175             'title': 'Uploads from Royalty Free Music - Topic',
5176             'uploader': 'Royalty Free Music - Topic',
5177             'modified_date': r're:\d{8}',
5178             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5179             'description': '',
5180             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5181             'tags': [],
5182             'channel': 'Royalty Free Music - Topic',
5183             'view_count': int,
5184             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5185         },
5186         'expected_warnings': [
5187             'does not have a videos tab',
5188             r'[Uu]navailable videos (are|will be) hidden',
5189         ],
5190         'playlist_mincount': 101,
5191         'params': {
5192             'skip_download': True,
5193             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5194         },
5195     }, {
5196         'note': 'non-standard redirect to regional channel',
5197         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5198         'only_matching': True
5199     }, {
5200         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5201         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5202         'info_dict': {
5203             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5204             'modified_date': '20220407',
5205             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5206             'tags': [],
5207             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5208             'uploader': 'pukkandan',
5209             'availability': 'unlisted',
5210             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5211             'channel': 'pukkandan',
5212             'description': 'Test for collaborative playlist',
5213             'title': 'yt-dlp test - collaborative playlist',
5214             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5215         },
5216         'playlist_mincount': 2
5217     }]
5218
5219     @classmethod
5220     def suitable(cls, url):
5221         return False if YoutubeIE.suitable(url) else super().suitable(url)
5222
5223     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5224
5225     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5226     def _real_extract(self, url, smuggled_data):
5227         item_id = self._match_id(url)
5228         url = compat_urlparse.urlunparse(
5229             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
5230         compat_opts = self.get_param('compat_opts', [])
5231
5232         def get_mobj(url):
5233             mobj = self._URL_RE.match(url).groupdict()
5234             mobj.update((k, '') for k, v in mobj.items() if v is None)
5235             return mobj
5236
5237         mobj, redirect_warning = get_mobj(url), None
5238         # Youtube returns incomplete data if tabname is not lower case
5239         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5240         if is_channel:
5241             if smuggled_data.get('is_music_url'):
5242                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5243                     item_id = item_id[2:]
5244                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5245                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5246                     mdata = self._extract_tab_endpoint(
5247                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5248                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5249                                         get_all=False, expected_type=compat_str)
5250                     if not murl:
5251                         raise ExtractorError('Failed to resolve album to playlist')
5252                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5253                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5254                     pre = f'https://www.youtube.com/channel/{item_id}'
5255
5256         original_tab_name = tab
5257         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5258             # Home URLs should redirect to /videos/
5259             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5260                                 'To download only the videos in the home page, add a "/featured" to the URL')
5261             tab = '/videos'
5262
5263         url = ''.join((pre, tab, post))
5264         mobj = get_mobj(url)
5265
5266         # Handle both video/playlist URLs
5267         qs = parse_qs(url)
5268         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5269
5270         if not video_id and mobj['not_channel'].startswith('watch'):
5271             if not playlist_id:
5272                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5273                 raise ExtractorError('Unable to recognize tab page')
5274             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5275             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5276             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5277             mobj = get_mobj(url)
5278
5279         if video_id and playlist_id:
5280             if self.get_param('noplaylist'):
5281                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5282                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5283                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5284             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5285
5286         data, ytcfg = self._extract_data(url, item_id)
5287
5288         # YouTube may provide a non-standard redirect to the regional channel
5289         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5290         redirect_url = traverse_obj(
5291             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5292         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5293             redirect_url = ''.join((
5294                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5295             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5296             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5297
5298         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5299         if tabs:
5300             selected_tab = self._extract_selected_tab(tabs)
5301             selected_tab_name = selected_tab.get('title', '').lower()
5302             if selected_tab_name == 'home':
5303                 selected_tab_name = 'featured'
5304             requested_tab_name = mobj['tab'][1:]
5305             if 'no-youtube-channel-redirect' not in compat_opts:
5306                 if requested_tab_name == 'live':
5307                     # Live tab should have redirected to the video
5308                     raise ExtractorError('The channel is not currently live', expected=True)
5309                 if requested_tab_name not in ('', selected_tab_name):
5310                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5311                     if not original_tab_name:
5312                         if item_id[:2] == 'UC':
5313                             # Topic channels don't have /videos. Use the equivalent playlist instead
5314                             pl_id = f'UU{item_id[2:]}'
5315                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5316                             try:
5317                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5318                             except ExtractorError:
5319                                 redirect_warning += ' and the playlist redirect gave error'
5320                             else:
5321                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5322                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5323                         if selected_tab_name and selected_tab_name != requested_tab_name:
5324                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5325                     else:
5326                         raise ExtractorError(redirect_warning, expected=True)
5327
5328         if redirect_warning:
5329             self.to_screen(redirect_warning)
5330         self.write_debug(f'Final URL: {url}')
5331
5332         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5333         if 'no-youtube-unavailable-videos' not in compat_opts:
5334             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5335         self._extract_and_report_alerts(data, only_once=True)
5336         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5337         if tabs:
5338             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5339
5340         playlist = traverse_obj(
5341             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5342         if playlist:
5343             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5344
5345         video_id = traverse_obj(
5346             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5347         if video_id:
5348             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5349                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5350             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5351                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5352
5353         raise ExtractorError('Unable to recognize tab page')
5354
5355
5356 class YoutubePlaylistIE(InfoExtractor):
5357     IE_DESC = 'YouTube playlists'
5358     _VALID_URL = r'''(?x)(?:
5359                         (?:https?://)?
5360                         (?:\w+\.)?
5361                         (?:
5362                             (?:
5363                                 youtube(?:kids)?\.com|
5364                                 %(invidious)s
5365                             )
5366                             /.*?\?.*?\blist=
5367                         )?
5368                         (?P<id>%(playlist_id)s)
5369                      )''' % {
5370         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5371         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5372     }
5373     IE_NAME = 'youtube:playlist'
5374     _TESTS = [{
5375         'note': 'issue #673',
5376         'url': 'PLBB231211A4F62143',
5377         'info_dict': {
5378             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5379             'id': 'PLBB231211A4F62143',
5380             'uploader': 'Wickman',
5381             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5382             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5383             'view_count': int,
5384             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5385             'modified_date': r're:\d{8}',
5386             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5387             'channel': 'Wickman',
5388             'tags': [],
5389             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5390         },
5391         'playlist_mincount': 29,
5392     }, {
5393         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5394         'info_dict': {
5395             'title': 'YDL_safe_search',
5396             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5397         },
5398         'playlist_count': 2,
5399         'skip': 'This playlist is private',
5400     }, {
5401         'note': 'embedded',
5402         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5403         'playlist_count': 4,
5404         'info_dict': {
5405             'title': 'JODA15',
5406             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5407             'uploader': 'milan',
5408             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5409             'description': '',
5410             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5411             'tags': [],
5412             'modified_date': '20140919',
5413             'view_count': int,
5414             'channel': 'milan',
5415             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5416             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5417         },
5418         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5419     }, {
5420         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5421         'playlist_mincount': 654,
5422         'info_dict': {
5423             'title': '2018 Chinese New Singles (11/6 updated)',
5424             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5425             'uploader': 'LBK',
5426             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5427             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5428             'channel': 'LBK',
5429             'view_count': int,
5430             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5431             'tags': [],
5432             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5433             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5434             'modified_date': r're:\d{8}',
5435         },
5436         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5437     }, {
5438         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5439         'only_matching': True,
5440     }, {
5441         # music album playlist
5442         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5443         'only_matching': True,
5444     }]
5445
5446     @classmethod
5447     def suitable(cls, url):
5448         if YoutubeTabIE.suitable(url):
5449             return False
5450         from ..utils import parse_qs
5451         qs = parse_qs(url)
5452         if qs.get('v', [None])[0]:
5453             return False
5454         return super().suitable(url)
5455
5456     def _real_extract(self, url):
5457         playlist_id = self._match_id(url)
5458         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5459         url = update_url_query(
5460             'https://www.youtube.com/playlist',
5461             parse_qs(url) or {'list': playlist_id})
5462         if is_music_url:
5463             url = smuggle_url(url, {'is_music_url': True})
5464         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5465
5466
5467 class YoutubeYtBeIE(InfoExtractor):
5468     IE_DESC = 'youtu.be'
5469     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5470     _TESTS = [{
5471         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5472         'info_dict': {
5473             'id': 'yeWKywCrFtk',
5474             'ext': 'mp4',
5475             'title': 'Small Scale Baler and Braiding Rugs',
5476             'uploader': 'Backus-Page House Museum',
5477             'uploader_id': 'backuspagemuseum',
5478             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5479             'upload_date': '20161008',
5480             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5481             'categories': ['Nonprofits & Activism'],
5482             'tags': list,
5483             'like_count': int,
5484             'age_limit': 0,
5485             'playable_in_embed': True,
5486             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5487             'channel': 'Backus-Page House Museum',
5488             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5489             'live_status': 'not_live',
5490             'view_count': int,
5491             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5492             'availability': 'public',
5493             'duration': 59,
5494         },
5495         'params': {
5496             'noplaylist': True,
5497             'skip_download': True,
5498         },
5499     }, {
5500         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5501         'only_matching': True,
5502     }]
5503
5504     def _real_extract(self, url):
5505         mobj = self._match_valid_url(url)
5506         video_id = mobj.group('id')
5507         playlist_id = mobj.group('playlist_id')
5508         return self.url_result(
5509             update_url_query('https://www.youtube.com/watch', {
5510                 'v': video_id,
5511                 'list': playlist_id,
5512                 'feature': 'youtu.be',
5513             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5514
5515
5516 class YoutubeLivestreamEmbedIE(InfoExtractor):
5517     IE_DESC = 'YouTube livestream embeds'
5518     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5519     _TESTS = [{
5520         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5521         'only_matching': True,
5522     }]
5523
5524     def _real_extract(self, url):
5525         channel_id = self._match_id(url)
5526         return self.url_result(
5527             f'https://www.youtube.com/channel/{channel_id}/live',
5528             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5529
5530
5531 class YoutubeYtUserIE(InfoExtractor):
5532     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5533     IE_NAME = 'youtube:user'
5534     _VALID_URL = r'ytuser:(?P<id>.+)'
5535     _TESTS = [{
5536         'url': 'ytuser:phihag',
5537         'only_matching': True,
5538     }]
5539
5540     def _real_extract(self, url):
5541         user_id = self._match_id(url)
5542         return self.url_result(
5543             'https://www.youtube.com/user/%s/videos' % user_id,
5544             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5545
5546
5547 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5548     IE_NAME = 'youtube:favorites'
5549     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5550     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5551     _LOGIN_REQUIRED = True
5552     _TESTS = [{
5553         'url': ':ytfav',
5554         'only_matching': True,
5555     }, {
5556         'url': ':ytfavorites',
5557         'only_matching': True,
5558     }]
5559
5560     def _real_extract(self, url):
5561         return self.url_result(
5562             'https://www.youtube.com/playlist?list=LL',
5563             ie=YoutubeTabIE.ie_key())
5564
5565
5566 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5567     IE_NAME = 'youtube:notif'
5568     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5569     _VALID_URL = r':ytnotif(?:ication)?s?'
5570     _LOGIN_REQUIRED = True
5571     _TESTS = [{
5572         'url': ':ytnotif',
5573         'only_matching': True,
5574     }, {
5575         'url': ':ytnotifications',
5576         'only_matching': True,
5577     }]
5578
5579     def _extract_notification_menu(self, response, continuation_list):
5580         notification_list = traverse_obj(
5581             response,
5582             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5583             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5584             expected_type=list) or []
5585         continuation_list[0] = None
5586         for item in notification_list:
5587             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5588             if entry:
5589                 yield entry
5590             continuation = item.get('continuationItemRenderer')
5591             if continuation:
5592                 continuation_list[0] = continuation
5593
5594     def _extract_notification_renderer(self, notification):
5595         video_id = traverse_obj(
5596             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5597         url = f'https://www.youtube.com/watch?v={video_id}'
5598         channel_id = None
5599         if not video_id:
5600             browse_ep = traverse_obj(
5601                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5602             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5603             post_id = self._search_regex(
5604                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5605                 'post id', default=None)
5606             if not channel_id or not post_id:
5607                 return
5608             # The direct /post url redirects to this in the browser
5609             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5610
5611         channel = traverse_obj(
5612             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5613             expected_type=str)
5614         title = self._search_regex(
5615             rf'{re.escape(channel)} [^:]+: (.+)', self._get_text(notification, 'shortMessage'),
5616             'video title', default=None)
5617         if title:
5618             title = title.replace('\xad', '')  # remove soft hyphens
5619         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5620                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5621                        else None)
5622         return {
5623             '_type': 'url',
5624             'url': url,
5625             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5626             'video_id': video_id,
5627             'title': title,
5628             'channel_id': channel_id,
5629             'channel': channel,
5630             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5631             'upload_date': upload_date,
5632         }
5633
5634     def _notification_menu_entries(self, ytcfg):
5635         continuation_list = [None]
5636         response = None
5637         for page in itertools.count(1):
5638             ctoken = traverse_obj(
5639                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5640             response = self._extract_response(
5641                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5642                 ep='notification/get_notification_menu', check_get_keys='actions',
5643                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5644             yield from self._extract_notification_menu(response, continuation_list)
5645             if not continuation_list[0]:
5646                 break
5647
5648     def _real_extract(self, url):
5649         display_id = 'notifications'
5650         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5651         self._report_playlist_authcheck(ytcfg)
5652         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5653
5654
5655 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5656     IE_DESC = 'YouTube search'
5657     IE_NAME = 'youtube:search'
5658     _SEARCH_KEY = 'ytsearch'
5659     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5660     _TESTS = [{
5661         'url': 'ytsearch5:youtube-dl test video',
5662         'playlist_count': 5,
5663         'info_dict': {
5664             'id': 'youtube-dl test video',
5665             'title': 'youtube-dl test video',
5666         }
5667     }]
5668
5669
5670 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5671     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5672     _SEARCH_KEY = 'ytsearchdate'
5673     IE_DESC = 'YouTube search, newest videos first'
5674     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5675     _TESTS = [{
5676         'url': 'ytsearchdate5:youtube-dl test video',
5677         'playlist_count': 5,
5678         'info_dict': {
5679             'id': 'youtube-dl test video',
5680             'title': 'youtube-dl test video',
5681         }
5682     }]
5683
5684
5685 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5686     IE_DESC = 'YouTube search URLs with sorting and filter support'
5687     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5688     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5689     _TESTS = [{
5690         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5691         'playlist_mincount': 5,
5692         'info_dict': {
5693             'id': 'youtube-dl test video',
5694             'title': 'youtube-dl test video',
5695         }
5696     }, {
5697         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5698         'playlist_mincount': 5,
5699         'info_dict': {
5700             'id': 'python',
5701             'title': 'python',
5702         }
5703     }, {
5704         'url': 'https://www.youtube.com/results?search_query=%23cats',
5705         'playlist_mincount': 1,
5706         'info_dict': {
5707             'id': '#cats',
5708             'title': '#cats',
5709             'entries': [{
5710                 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5711                 'title': '#cats',
5712             }],
5713         },
5714     }, {
5715         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5716         'only_matching': True,
5717     }]
5718
5719     def _real_extract(self, url):
5720         qs = parse_qs(url)
5721         query = (qs.get('search_query') or qs.get('q'))[0]
5722         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5723
5724
5725 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5726     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5727     IE_NAME = 'youtube:music:search_url'
5728     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5729     _TESTS = [{
5730         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5731         'playlist_count': 16,
5732         'info_dict': {
5733             'id': 'royalty free music',
5734             'title': 'royalty free music',
5735         }
5736     }, {
5737         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5738         'playlist_mincount': 30,
5739         'info_dict': {
5740             'id': 'royalty free music - songs',
5741             'title': 'royalty free music - songs',
5742         },
5743         'params': {'extract_flat': 'in_playlist'}
5744     }, {
5745         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5746         'playlist_mincount': 30,
5747         'info_dict': {
5748             'id': 'royalty free music - community playlists',
5749             'title': 'royalty free music - community playlists',
5750         },
5751         'params': {'extract_flat': 'in_playlist'}
5752     }]
5753
5754     _SECTIONS = {
5755         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5756         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5757         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5758         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5759         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5760         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5761     }
5762
5763     def _real_extract(self, url):
5764         qs = parse_qs(url)
5765         query = (qs.get('search_query') or qs.get('q'))[0]
5766         params = qs.get('sp', (None,))[0]
5767         if params:
5768             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5769         else:
5770             section = compat_urllib_parse_unquote_plus((url.split('#') + [''])[1]).lower()
5771             params = self._SECTIONS.get(section)
5772             if not params:
5773                 section = None
5774         title = join_nonempty(query, section, delim=' - ')
5775         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5776
5777
5778 class YoutubeFeedsInfoExtractor(InfoExtractor):
5779     """
5780     Base class for feed extractors
5781     Subclasses must define the _FEED_NAME property.
5782     """
5783     _LOGIN_REQUIRED = True
5784
5785     def _real_initialize(self):
5786         YoutubeBaseInfoExtractor._check_login_required(self)
5787
5788     @property
5789     def IE_NAME(self):
5790         return 'youtube:%s' % self._FEED_NAME
5791
5792     def _real_extract(self, url):
5793         return self.url_result(
5794             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5795
5796
5797 class YoutubeWatchLaterIE(InfoExtractor):
5798     IE_NAME = 'youtube:watchlater'
5799     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5800     _VALID_URL = r':ytwatchlater'
5801     _TESTS = [{
5802         'url': ':ytwatchlater',
5803         'only_matching': True,
5804     }]
5805
5806     def _real_extract(self, url):
5807         return self.url_result(
5808             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5809
5810
5811 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5812     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5813     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5814     _FEED_NAME = 'recommended'
5815     _LOGIN_REQUIRED = False
5816     _TESTS = [{
5817         'url': ':ytrec',
5818         'only_matching': True,
5819     }, {
5820         'url': ':ytrecommended',
5821         'only_matching': True,
5822     }, {
5823         'url': 'https://youtube.com',
5824         'only_matching': True,
5825     }]
5826
5827
5828 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5829     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5830     _VALID_URL = r':ytsub(?:scription)?s?'
5831     _FEED_NAME = 'subscriptions'
5832     _TESTS = [{
5833         'url': ':ytsubs',
5834         'only_matching': True,
5835     }, {
5836         'url': ':ytsubscriptions',
5837         'only_matching': True,
5838     }]
5839
5840
5841 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5842     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5843     _VALID_URL = r':ythis(?:tory)?'
5844     _FEED_NAME = 'history'
5845     _TESTS = [{
5846         'url': ':ythistory',
5847         'only_matching': True,
5848     }]
5849
5850
5851 class YoutubeStoriesIE(InfoExtractor):
5852     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5853     IE_NAME = 'youtube:stories'
5854     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5855     _TESTS = [{
5856         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5857         'only_matching': True,
5858     }]
5859
5860     def _real_extract(self, url):
5861         playlist_id = f'RLTD{self._match_id(url)}'
5862         return self.url_result(
5863             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5864             ie=YoutubeTabIE, video_id=playlist_id)
5865
5866
5867 class YoutubeTruncatedURLIE(InfoExtractor):
5868     IE_NAME = 'youtube:truncated_url'
5869     IE_DESC = False  # Do not list
5870     _VALID_URL = r'''(?x)
5871         (?:https?://)?
5872         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5873         (?:watch\?(?:
5874             feature=[a-z_]+|
5875             annotation_id=annotation_[^&]+|
5876             x-yt-cl=[0-9]+|
5877             hl=[^&]*|
5878             t=[0-9]+
5879         )?
5880         |
5881             attribution_link\?a=[^&]+
5882         )
5883         $
5884     '''
5885
5886     _TESTS = [{
5887         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5888         'only_matching': True,
5889     }, {
5890         'url': 'https://www.youtube.com/watch?',
5891         'only_matching': True,
5892     }, {
5893         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5894         'only_matching': True,
5895     }, {
5896         'url': 'https://www.youtube.com/watch?feature=foo',
5897         'only_matching': True,
5898     }, {
5899         'url': 'https://www.youtube.com/watch?hl=en-GB',
5900         'only_matching': True,
5901     }, {
5902         'url': 'https://www.youtube.com/watch?t=2372',
5903         'only_matching': True,
5904     }]
5905
5906     def _real_extract(self, url):
5907         raise ExtractorError(
5908             'Did you forget to quote the URL? Remember that & is a meta '
5909             'character in most shells, so you want to put the URL in quotes, '
5910             'like  youtube-dl '
5911             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5912             ' or simply  youtube-dl BaW_jenozKc  .',
5913             expected=True)
5914
5915
5916 class YoutubeClipIE(InfoExtractor):
5917     IE_NAME = 'youtube:clip'
5918     IE_DESC = False  # Do not list
5919     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/'
5920
5921     def _real_extract(self, url):
5922         self.report_warning('YouTube clips are not currently supported. The entire video will be downloaded instead')
5923         return self.url_result(url, 'Generic')
5924
5925
5926 class YoutubeTruncatedIDIE(InfoExtractor):
5927     IE_NAME = 'youtube:truncated_id'
5928     IE_DESC = False  # Do not list
5929     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
5930
5931     _TESTS = [{
5932         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
5933         'only_matching': True,
5934     }]
5935
5936     def _real_extract(self, url):
5937         video_id = self._match_id(url)
5938         raise ExtractorError(
5939             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
5940             expected=True)