yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import functools
   6 import hashlib
   7 import itertools
   8 import json
   9 import math
  10 import os.path
  11 import random
  12 import re
  13 import sys
  14 import threading
  15 import time
  16 import traceback
  17
  18 from .common import InfoExtractor, SearchInfoExtractor
  19 from ..compat import (
  20     compat_chr,
  21     compat_HTTPError,
  22     compat_parse_qs,
  23     compat_str,
  24     compat_urllib_parse_unquote_plus,
  25     compat_urllib_parse_urlencode,
  26     compat_urllib_parse_urlparse,
  27     compat_urlparse,
  28 )
  29 from ..jsinterp import JSInterpreter
  30 from ..utils import (
  31     NO_DEFAULT,
  32     ExtractorError,
  33     bug_reports_message,
  34     clean_html,
  35     datetime_from_str,
  36     dict_get,
  37     error_to_compat_str,
  38     float_or_none,
  39     format_field,
  40     get_first,
  41     int_or_none,
  42     is_html,
  43     join_nonempty,
  44     js_to_json,
  45     mimetype2ext,
  46     network_exceptions,
  47     orderedSet,
  48     parse_codecs,
  49     parse_count,
  50     parse_duration,
  51     parse_iso8601,
  52     parse_qs,
  53     qualities,
  54     remove_end,
  55     remove_start,
  56     smuggle_url,
  57     str_or_none,
  58     str_to_int,
  59     strftime_or_none,
  60     traverse_obj,
  61     try_get,
  62     unescapeHTML,
  63     unified_strdate,
  64     unified_timestamp,
  65     unsmuggle_url,
  66     update_url_query,
  67     url_or_none,
  68     urljoin,
  69     variadic,
  70 )
  71
  72 # any clients starting with _ cannot be explicity requested by the user
  73 INNERTUBE_CLIENTS = {
  74     'web': {
  75         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  76         'INNERTUBE_CONTEXT': {
  77             'client': {
  78                 'clientName': 'WEB',
  79                 'clientVersion': '2.20211221.00.00',
  80             }
  81         },
  82         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  83     },
  84     'web_embedded': {
  85         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  86         'INNERTUBE_CONTEXT': {
  87             'client': {
  88                 'clientName': 'WEB_EMBEDDED_PLAYER',
  89                 'clientVersion': '1.20211215.00.01',
  90             },
  91         },
  92         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  93     },
  94     'web_music': {
  95         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  96         'INNERTUBE_HOST': 'music.youtube.com',
  97         'INNERTUBE_CONTEXT': {
  98             'client': {
  99                 'clientName': 'WEB_REMIX',
 100                 'clientVersion': '1.20211213.00.00',
 101             }
 102         },
 103         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 104     },
 105     'web_creator': {
 106         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 107         'INNERTUBE_CONTEXT': {
 108             'client': {
 109                 'clientName': 'WEB_CREATOR',
 110                 'clientVersion': '1.20211220.02.00',
 111             }
 112         },
 113         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 114     },
 115     'android': {
 116         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 117         'INNERTUBE_CONTEXT': {
 118             'client': {
 119                 'clientName': 'ANDROID',
 120                 'clientVersion': '16.49',
 121             }
 122         },
 123         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 124         'REQUIRE_JS_PLAYER': False
 125     },
 126     'android_embedded': {
 127         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 128         'INNERTUBE_CONTEXT': {
 129             'client': {
 130                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 131                 'clientVersion': '16.49',
 132             },
 133         },
 134         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 135         'REQUIRE_JS_PLAYER': False
 136     },
 137     'android_music': {
 138         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 139         'INNERTUBE_CONTEXT': {
 140             'client': {
 141                 'clientName': 'ANDROID_MUSIC',
 142                 'clientVersion': '4.57',
 143             }
 144         },
 145         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 146         'REQUIRE_JS_PLAYER': False
 147     },
 148     'android_creator': {
 149         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 150         'INNERTUBE_CONTEXT': {
 151             'client': {
 152                 'clientName': 'ANDROID_CREATOR',
 153                 'clientVersion': '21.47',
 154             },
 155         },
 156         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 157         'REQUIRE_JS_PLAYER': False
 158     },
 159     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 160     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 161     'ios': {
 162         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 163         'INNERTUBE_CONTEXT': {
 164             'client': {
 165                 'clientName': 'IOS',
 166                 'clientVersion': '16.46',
 167                 'deviceModel': 'iPhone14,3',
 168             }
 169         },
 170         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 171         'REQUIRE_JS_PLAYER': False
 172     },
 173     'ios_embedded': {
 174         'INNERTUBE_CONTEXT': {
 175             'client': {
 176                 'clientName': 'IOS_MESSAGES_EXTENSION',
 177                 'clientVersion': '16.46',
 178                 'deviceModel': 'iPhone14,3',
 179             },
 180         },
 181         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 182         'REQUIRE_JS_PLAYER': False
 183     },
 184     'ios_music': {
 185         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 186         'INNERTUBE_CONTEXT': {
 187             'client': {
 188                 'clientName': 'IOS_MUSIC',
 189                 'clientVersion': '4.57',
 190             },
 191         },
 192         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 193         'REQUIRE_JS_PLAYER': False
 194     },
 195     'ios_creator': {
 196         'INNERTUBE_CONTEXT': {
 197             'client': {
 198                 'clientName': 'IOS_CREATOR',
 199                 'clientVersion': '21.47',
 200             },
 201         },
 202         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 203         'REQUIRE_JS_PLAYER': False
 204     },
 205     # mweb has 'ultralow' formats
 206     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 207     'mweb': {
 208         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 209         'INNERTUBE_CONTEXT': {
 210             'client': {
 211                 'clientName': 'MWEB',
 212                 'clientVersion': '2.20211221.01.00',
 213             }
 214         },
 215         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 216     },
 217     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 218     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 219     'tv_embedded': {
 220         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 221         'INNERTUBE_CONTEXT': {
 222             'client': {
 223                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 224                 'clientVersion': '2.0',
 225             },
 226         },
 227         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 228     },
 229 }
 230
 231
 232 def _split_innertube_client(client_name):
 233     variant, *base = client_name.rsplit('.', 1)
 234     if base:
 235         return variant, base[0], variant
 236     base, *variant = client_name.split('_', 1)
 237     return client_name, base, variant[0] if variant else None
 238
 239
 240 def build_innertube_clients():
 241     THIRD_PARTY = {
 242         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 243     }
 244     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 245     priority = qualities(BASE_CLIENTS[::-1])
 246
 247     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 248         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 249         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 250         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 251         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 252
 253         _, base_client, variant = _split_innertube_client(client)
 254         ytcfg['priority'] = 10 * priority(base_client)
 255
 256         if not variant:
 257             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 258             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 259             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 260             embedscreen['priority'] -= 3
 261         elif variant == 'embedded':
 262             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 263             ytcfg['priority'] -= 2
 264         else:
 265             ytcfg['priority'] -= 3
 266
 267
 268 build_innertube_clients()
 269
 270
 271 class YoutubeBaseInfoExtractor(InfoExtractor):
 272     """Provide base functions for Youtube extractors"""
 273
 274     _RESERVED_NAMES = (
 275         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 276         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 277         r'browse|oembed|get_video_info|iframe_api|s/player|'
 278         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 279
 280     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 281
 282     # _NETRC_MACHINE = 'youtube'
 283
 284     # If True it will raise an error if no login info is provided
 285     _LOGIN_REQUIRED = False
 286
 287     _INVIDIOUS_SITES = (
 288         # invidious-redirect websites
 289         r'(?:www\.)?redirect\.invidious\.io',
 290         r'(?:(?:www|dev)\.)?invidio\.us',
 291         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 292         r'(?:www\.)?invidious\.pussthecat\.org',
 293         r'(?:www\.)?invidious\.zee\.li',
 294         r'(?:www\.)?invidious\.ethibox\.fr',
 295         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 296         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 297         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 298         # youtube-dl invidious instances list
 299         r'(?:(?:www|no)\.)?invidiou\.sh',
 300         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 301         r'(?:www\.)?invidious\.kabi\.tk',
 302         r'(?:www\.)?invidious\.mastodon\.host',
 303         r'(?:www\.)?invidious\.zapashcanon\.fr',
 304         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 305         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 306         r'(?:www\.)?invidious\.himiko\.cloud',
 307         r'(?:www\.)?invidious\.reallyancient\.tech',
 308         r'(?:www\.)?invidious\.tube',
 309         r'(?:www\.)?invidiou\.site',
 310         r'(?:www\.)?invidious\.site',
 311         r'(?:www\.)?invidious\.xyz',
 312         r'(?:www\.)?invidious\.nixnet\.xyz',
 313         r'(?:www\.)?invidious\.048596\.xyz',
 314         r'(?:www\.)?invidious\.drycat\.fr',
 315         r'(?:www\.)?inv\.skyn3t\.in',
 316         r'(?:www\.)?tube\.poal\.co',
 317         r'(?:www\.)?tube\.connect\.cafe',
 318         r'(?:www\.)?vid\.wxzm\.sx',
 319         r'(?:www\.)?vid\.mint\.lgbt',
 320         r'(?:www\.)?vid\.puffyan\.us',
 321         r'(?:www\.)?yewtu\.be',
 322         r'(?:www\.)?yt\.elukerio\.org',
 323         r'(?:www\.)?yt\.lelux\.fi',
 324         r'(?:www\.)?invidious\.ggc-project\.de',
 325         r'(?:www\.)?yt\.maisputain\.ovh',
 326         r'(?:www\.)?ytprivate\.com',
 327         r'(?:www\.)?invidious\.13ad\.de',
 328         r'(?:www\.)?invidious\.toot\.koeln',
 329         r'(?:www\.)?invidious\.fdn\.fr',
 330         r'(?:www\.)?watch\.nettohikari\.com',
 331         r'(?:www\.)?invidious\.namazso\.eu',
 332         r'(?:www\.)?invidious\.silkky\.cloud',
 333         r'(?:www\.)?invidious\.exonip\.de',
 334         r'(?:www\.)?invidious\.riverside\.rocks',
 335         r'(?:www\.)?invidious\.blamefran\.net',
 336         r'(?:www\.)?invidious\.moomoo\.de',
 337         r'(?:www\.)?ytb\.trom\.tf',
 338         r'(?:www\.)?yt\.cyberhost\.uk',
 339         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 340         r'(?:www\.)?qklhadlycap4cnod\.onion',
 341         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 342         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 343         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 344         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 345         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 346         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 347         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 348         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 349         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 350         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 351     )
 352
 353     def _initialize_consent(self):
 354         cookies = self._get_cookies('https://www.youtube.com/')
 355         if cookies.get('__Secure-3PSID'):
 356             return
 357         consent_id = None
 358         consent = cookies.get('CONSENT')
 359         if consent:
 360             if 'YES' in consent.value:
 361                 return
 362             consent_id = self._search_regex(
 363                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 364         if not consent_id:
 365             consent_id = random.randint(100, 999)
 366         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 367
 368     def _initialize_pref(self):
 369         cookies = self._get_cookies('https://www.youtube.com/')
 370         pref_cookie = cookies.get('PREF')
 371         pref = {}
 372         if pref_cookie:
 373             try:
 374                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 375             except ValueError:
 376                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 377         pref.update({'hl': 'en', 'tz': 'UTC'})
 378         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 379
 380     def _real_initialize(self):
 381         self._initialize_pref()
 382         self._initialize_consent()
 383         self._check_login_required()
 384
 385     def _check_login_required(self):
 386         if (self._LOGIN_REQUIRED
 387                 and self.get_param('cookiefile') is None
 388                 and self.get_param('cookiesfrombrowser') is None):
 389             self.raise_login_required('Login details are needed to download this content', method='cookies')
 390
 391     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 392     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 393     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 394
 395     def _get_default_ytcfg(self, client='web'):
 396         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 397
 398     def _get_innertube_host(self, client='web'):
 399         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 400
 401     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 402         # try_get but with fallback to default ytcfg client values when present
 403         _func = lambda y: try_get(y, getter, expected_type)
 404         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 405
 406     def _extract_client_name(self, ytcfg, default_client='web'):
 407         return self._ytcfg_get_safe(
 408             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 409                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 410
 411     def _extract_client_version(self, ytcfg, default_client='web'):
 412         return self._ytcfg_get_safe(
 413             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 414                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 415
 416     def _extract_api_key(self, ytcfg=None, default_client='web'):
 417         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 418
 419     def _extract_context(self, ytcfg=None, default_client='web'):
 420         context = get_first(
 421             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 422         # Enforce language and tz for extraction
 423         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 424         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 425         return context
 426
 427     _SAPISID = None
 428
 429     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 430         time_now = round(time.time())
 431         if self._SAPISID is None:
 432             yt_cookies = self._get_cookies('https://www.youtube.com')
 433             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 434             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 435             sapisid_cookie = dict_get(
 436                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 437             if sapisid_cookie and sapisid_cookie.value:
 438                 self._SAPISID = sapisid_cookie.value
 439                 self.write_debug('Extracted SAPISID cookie')
 440                 # SAPISID cookie is required if not already present
 441                 if not yt_cookies.get('SAPISID'):
 442                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 443                     self._set_cookie(
 444                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 445             else:
 446                 self._SAPISID = False
 447         if not self._SAPISID:
 448             return None
 449         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 450         sapisidhash = hashlib.sha1(
 451             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 452         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 453
 454     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 455                   note='Downloading API JSON', errnote='Unable to download API page',
 456                   context=None, api_key=None, api_hostname=None, default_client='web'):
 457
 458         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 459         data.update(query)
 460         real_headers = self.generate_api_headers(default_client=default_client)
 461         real_headers.update({'content-type': 'application/json'})
 462         if headers:
 463             real_headers.update(headers)
 464         return self._download_json(
 465             f'https://{api_hostname or self._get_innertube_host(default_client)}/youtubei/v1/{ep}',
 466             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 467             data=json.dumps(data).encode('utf8'), headers=real_headers,
 468             query={'key': api_key or self._extract_api_key(), 'prettyPrint': 'false'})
 469
 470     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 471         data = self._search_regex(
 472             (fr'{self._YT_INITIAL_DATA_RE}\s*{self._YT_INITIAL_BOUNDARY_RE}',
 473              self._YT_INITIAL_DATA_RE), webpage, 'yt initial data', fatal=fatal)
 474         if data:
 475             return self._parse_json(data, item_id, fatal=fatal)
 476
 477     @staticmethod
 478     def _extract_session_index(*data):
 479         """
 480         Index of current account in account list.
 481         See: https://github.com/yt-dlp/yt-dlp/pull/519
 482         """
 483         for ytcfg in data:
 484             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 485             if session_index is not None:
 486                 return session_index
 487
 488     # Deprecated?
 489     def _extract_identity_token(self, ytcfg=None, webpage=None):
 490         if ytcfg:
 491             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 492             if token:
 493                 return token
 494         if webpage:
 495             return self._search_regex(
 496                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 497                 'identity token', default=None, fatal=False)
 498
 499     @staticmethod
 500     def _extract_account_syncid(*args):
 501         """
 502         Extract syncId required to download private playlists of secondary channels
 503         @params response and/or ytcfg
 504         """
 505         for data in args:
 506             # ytcfg includes channel_syncid if on secondary channel
 507             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 508             if delegated_sid:
 509                 return delegated_sid
 510             sync_ids = (try_get(
 511                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 512                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 513             if len(sync_ids) >= 2 and sync_ids[1]:
 514                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 515                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 516                 return sync_ids[0]
 517
 518     @staticmethod
 519     def _extract_visitor_data(*args):
 520         """
 521         Extracts visitorData from an API response or ytcfg
 522         Appears to be used to track session state
 523         """
 524         return get_first(
 525             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 526             expected_type=str)
 527
 528     @property
 529     def is_authenticated(self):
 530         return bool(self._generate_sapisidhash_header())
 531
 532     def extract_ytcfg(self, video_id, webpage):
 533         if not webpage:
 534             return {}
 535         return self._parse_json(
 536             self._search_regex(
 537                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 538                 default='{}'), video_id, fatal=False) or {}
 539
 540     def generate_api_headers(
 541             self, *, ytcfg=None, account_syncid=None, session_index=None,
 542             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 543
 544         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
 545         headers = {
 546             'X-YouTube-Client-Name': compat_str(
 547                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 548             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 549             'Origin': origin,
 550             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 551             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 552             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 553         }
 554         if session_index is None:
 555             session_index = self._extract_session_index(ytcfg)
 556         if account_syncid or session_index is not None:
 557             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 558
 559         auth = self._generate_sapisidhash_header(origin)
 560         if auth is not None:
 561             headers['Authorization'] = auth
 562             headers['X-Origin'] = origin
 563         return {h: v for h, v in headers.items() if v is not None}
 564
 565     def _download_ytcfg(self, client, video_id):
 566         url = {
 567             'web': 'https://www.youtube.com',
 568             'web_music': 'https://music.youtube.com',
 569             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 570         }.get(client)
 571         if not url:
 572             return {}
 573         webpage = self._download_webpage(
 574             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 575         return self.extract_ytcfg(video_id, webpage) or {}
 576
 577     @staticmethod
 578     def _build_api_continuation_query(continuation, ctp=None):
 579         query = {
 580             'continuation': continuation
 581         }
 582         # TODO: Inconsistency with clickTrackingParams.
 583         # Currently we have a fixed ctp contained within context (from ytcfg)
 584         # and a ctp in root query for continuation.
 585         if ctp:
 586             query['clickTracking'] = {'clickTrackingParams': ctp}
 587         return query
 588
 589     @classmethod
 590     def _extract_next_continuation_data(cls, renderer):
 591         next_continuation = try_get(
 592             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 593                        lambda x: x['continuation']['reloadContinuationData']), dict)
 594         if not next_continuation:
 595             return
 596         continuation = next_continuation.get('continuation')
 597         if not continuation:
 598             return
 599         ctp = next_continuation.get('clickTrackingParams')
 600         return cls._build_api_continuation_query(continuation, ctp)
 601
 602     @classmethod
 603     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 604         if isinstance(continuation_ep, dict):
 605             continuation = try_get(
 606                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 607             if not continuation:
 608                 return
 609             ctp = continuation_ep.get('clickTrackingParams')
 610             return cls._build_api_continuation_query(continuation, ctp)
 611
 612     @classmethod
 613     def _extract_continuation(cls, renderer):
 614         next_continuation = cls._extract_next_continuation_data(renderer)
 615         if next_continuation:
 616             return next_continuation
 617
 618         contents = []
 619         for key in ('contents', 'items'):
 620             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 621
 622         for content in contents:
 623             if not isinstance(content, dict):
 624                 continue
 625             continuation_ep = try_get(
 626                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 627                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 628                 dict)
 629             continuation = cls._extract_continuation_ep_data(continuation_ep)
 630             if continuation:
 631                 return continuation
 632
 633     @classmethod
 634     def _extract_alerts(cls, data):
 635         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 636             if not isinstance(alert_dict, dict):
 637                 continue
 638             for alert in alert_dict.values():
 639                 alert_type = alert.get('type')
 640                 if not alert_type:
 641                     continue
 642                 message = cls._get_text(alert, 'text')
 643                 if message:
 644                     yield alert_type, message
 645
 646     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 647         errors = []
 648         warnings = []
 649         for alert_type, alert_message in alerts:
 650             if alert_type.lower() == 'error' and fatal:
 651                 errors.append([alert_type, alert_message])
 652             else:
 653                 warnings.append([alert_type, alert_message])
 654
 655         for alert_type, alert_message in (warnings + errors[:-1]):
 656             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 657         if errors:
 658             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 659
 660     def _extract_and_report_alerts(self, data, *args, **kwargs):
 661         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 662
 663     def _extract_badges(self, renderer: dict):
 664         badges = set()
 665         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 666             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 667             if label:
 668                 badges.add(label.lower())
 669         return badges
 670
 671     @staticmethod
 672     def _get_text(data, *path_list, max_runs=None):
 673         for path in path_list or [None]:
 674             if path is None:
 675                 obj = [data]
 676             else:
 677                 obj = traverse_obj(data, path, default=[])
 678                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 679                     obj = [obj]
 680             for item in obj:
 681                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 682                 if text:
 683                     return text
 684                 runs = try_get(item, lambda x: x['runs'], list) or []
 685                 if not runs and isinstance(item, list):
 686                     runs = item
 687
 688                 runs = runs[:min(len(runs), max_runs or len(runs))]
 689                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 690                 if text:
 691                     return text
 692
 693     def _get_count(self, data, *path_list):
 694         count_text = self._get_text(data, *path_list) or ''
 695         count = parse_count(count_text)
 696         if count is None:
 697             count = str_to_int(
 698                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 699         return count
 700
 701     @staticmethod
 702     def _extract_thumbnails(data, *path_list):
 703         """
 704         Extract thumbnails from thumbnails dict
 705         @param path_list: path list to level that contains 'thumbnails' key
 706         """
 707         thumbnails = []
 708         for path in path_list or [()]:
 709             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 710                 thumbnail_url = url_or_none(thumbnail.get('url'))
 711                 if not thumbnail_url:
 712                     continue
 713                 # Sometimes youtube gives a wrong thumbnail URL. See:
 714                 # https://github.com/yt-dlp/yt-dlp/issues/233
 715                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 716                 if 'maxresdefault' in thumbnail_url:
 717                     thumbnail_url = thumbnail_url.split('?')[0]
 718                 thumbnails.append({
 719                     'url': thumbnail_url,
 720                     'height': int_or_none(thumbnail.get('height')),
 721                     'width': int_or_none(thumbnail.get('width')),
 722                 })
 723         return thumbnails
 724
 725     @staticmethod
 726     def extract_relative_time(relative_time_text):
 727         """
 728         Extracts a relative time from string and converts to dt object
 729         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 730         """
 731         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 732         if mobj:
 733             start = mobj.group('start')
 734             if start:
 735                 return datetime_from_str(start)
 736             try:
 737                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 738             except ValueError:
 739                 return None
 740
 741     def _extract_time_text(self, renderer, *path_list):
 742         """@returns (timestamp, time_text)"""
 743         text = self._get_text(renderer, *path_list) or ''
 744         dt = self.extract_relative_time(text)
 745         timestamp = None
 746         if isinstance(dt, datetime.datetime):
 747             timestamp = calendar.timegm(dt.timetuple())
 748
 749         if timestamp is None:
 750             timestamp = (
 751                 unified_timestamp(text) or unified_timestamp(
 752                     self._search_regex(
 753                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 754                         text.lower(), 'time text', default=None)))
 755
 756         if text and timestamp is None:
 757             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 758         return timestamp, text
 759
 760     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 761                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 762                           default_client='web'):
 763         response = None
 764         last_error = None
 765         count = -1
 766         retries = self.get_param('extractor_retries', 3)
 767         if check_get_keys is None:
 768             check_get_keys = []
 769         while count < retries:
 770             count += 1
 771             if last_error:
 772                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 773             try:
 774                 response = self._call_api(
 775                     ep=ep, fatal=True, headers=headers,
 776                     video_id=item_id, query=query,
 777                     context=self._extract_context(ytcfg, default_client),
 778                     api_key=self._extract_api_key(ytcfg, default_client),
 779                     api_hostname=api_hostname, default_client=default_client,
 780                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 781             except ExtractorError as e:
 782                 if isinstance(e.cause, network_exceptions):
 783                     if isinstance(e.cause, compat_HTTPError):
 784                         first_bytes = e.cause.read(512)
 785                         if not is_html(first_bytes):
 786                             yt_error = try_get(
 787                                 self._parse_json(
 788                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 789                                 lambda x: x['error']['message'], compat_str)
 790                             if yt_error:
 791                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 792                     # Downloading page may result in intermittent 5xx HTTP error
 793                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 794                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 795                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 796                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 797                         last_error = error_to_compat_str(e.cause or e.msg)
 798                         if count < retries:
 799                             continue
 800                 if fatal:
 801                     raise
 802                 else:
 803                     self.report_warning(error_to_compat_str(e))
 804                     return
 805
 806             else:
 807                 try:
 808                     self._extract_and_report_alerts(response, only_once=True)
 809                 except ExtractorError as e:
 810                     # YouTube servers may return errors we want to retry on in a 200 OK response
 811                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 812                     if 'unknown error' in e.msg.lower():
 813                         last_error = e.msg
 814                         continue
 815                     if fatal:
 816                         raise
 817                     self.report_warning(error_to_compat_str(e))
 818                     return
 819                 if not check_get_keys or dict_get(response, check_get_keys):
 820                     break
 821                 # Youtube sometimes sends incomplete data
 822                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 823                 last_error = 'Incomplete data received'
 824                 if count >= retries:
 825                     if fatal:
 826                         raise ExtractorError(last_error)
 827                     else:
 828                         self.report_warning(last_error)
 829                         return
 830         return response
 831
 832     @staticmethod
 833     def is_music_url(url):
 834         return re.match(r'https?://music\.youtube\.com/', url) is not None
 835
 836     def _extract_video(self, renderer):
 837         video_id = renderer.get('videoId')
 838         title = self._get_text(renderer, 'title')
 839         description = self._get_text(renderer, 'descriptionSnippet')
 840         duration = parse_duration(self._get_text(
 841             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 842         if duration is None:
 843             duration = parse_duration(self._search_regex(
 844                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 845                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 846                 video_id, default=None, group='duration'))
 847
 848         view_count = self._get_count(renderer, 'viewCountText')
 849
 850         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 851         channel_id = traverse_obj(
 852             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 853             expected_type=str, get_all=False)
 854         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 855         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 856         overlay_style = traverse_obj(
 857             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 858             get_all=False, expected_type=str)
 859         badges = self._extract_badges(renderer)
 860         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 861         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 862             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 863             expected_type=str)) or ''
 864         url = f'https://www.youtube.com/watch?v={video_id}'
 865         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 866             url = f'https://www.youtube.com/shorts/{video_id}'
 867
 868         return {
 869             '_type': 'url',
 870             'ie_key': YoutubeIE.ie_key(),
 871             'id': video_id,
 872             'url': url,
 873             'title': title,
 874             'description': description,
 875             'duration': duration,
 876             'view_count': view_count,
 877             'uploader': uploader,
 878             'channel_id': channel_id,
 879             'thumbnails': thumbnails,
 880             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 881                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 882                             else None),
 883             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 884                             else 'was_live' if 'streamed' in time_text.lower()
 885                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 886                             else None),
 887             'release_timestamp': scheduled_timestamp,
 888             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 889         }
 890
 891
 892 class YoutubeIE(YoutubeBaseInfoExtractor):
 893     IE_DESC = 'YouTube'
 894     _VALID_URL = r"""(?x)^
 895                      (
 896                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 897                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 898                             (?:www\.)?deturl\.com/www\.youtube\.com|
 899                             (?:www\.)?pwnyoutube\.com|
 900                             (?:www\.)?hooktube\.com|
 901                             (?:www\.)?yourepeat\.com|
 902                             tube\.majestyc\.net|
 903                             %(invidious)s|
 904                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 905                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 906                          (?:                                                  # the various things that can precede the ID:
 907                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 908                              |(?:                                             # or the v= param in all its forms
 909                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 910                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 911                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 912                                  v=
 913                              )
 914                          ))
 915                          |(?:
 916                             youtu\.be|                                        # just youtu.be/xxxx
 917                             vid\.plus|                                        # or vid.plus/xxxx
 918                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 919                             %(invidious)s
 920                          )/
 921                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 922                          )
 923                      )?                                                       # all until now is optional -> you can pass the naked ID
 924                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 925                      (?(1).+)?                                                # if we found the ID, everything can follow
 926                      (?:\#|$)""" % {
 927         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 928     }
 929     _PLAYER_INFO_RE = (
 930         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 931         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 932         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 933     )
 934     _formats = {
 935         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 936         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 937         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 938         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 939         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 940         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 941         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 942         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 943         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 944         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 945         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 946         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 947         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 948         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 949         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 950         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 951         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 953
 954
 955         # 3D videos
 956         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 957         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 958         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 959         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 960         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 961         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 962         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 963
 964         # Apple HTTP Live Streaming
 965         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 966         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 967         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 968         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 969         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 970         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 971         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 972         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 973
 974         # DASH mp4 video
 975         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 976         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 977         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 978         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 979         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 980         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 981         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 982         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 983         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 984         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 985         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 986         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987
 988         # Dash mp4 audio
 989         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 990         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 991         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 992         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 993         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 994         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 995         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 996
 997         # Dash webm
 998         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 999         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1000         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1001         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1002         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1003         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1004         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1005         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1006         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1007         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1008         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1009         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1010         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1011         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1012         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1013         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1014         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1015         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1016         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1017         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1018         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1020
1021         # Dash webm audio
1022         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1023         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1024
1025         # Dash webm audio with opus inside
1026         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1027         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1028         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1029
1030         # RTMP (unnamed)
1031         '_rtmp': {'protocol': 'rtmp'},
1032
1033         # av01 video only formats sometimes served with "unknown" codecs
1034         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1035         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1036         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1037         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1038         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1039         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1040         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1041         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1042     }
1043     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1044
1045     _GEO_BYPASS = False
1046
1047     IE_NAME = 'youtube'
1048     _TESTS = [
1049         {
1050             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1051             'info_dict': {
1052                 'id': 'BaW_jenozKc',
1053                 'ext': 'mp4',
1054                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1055                 'uploader': 'Philipp Hagemeister',
1056                 'uploader_id': 'phihag',
1057                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1058                 'channel': 'Philipp Hagemeister',
1059                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1060                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1061                 'upload_date': '20121002',
1062                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1063                 'categories': ['Science & Technology'],
1064                 'tags': ['youtube-dl'],
1065                 'duration': 10,
1066                 'view_count': int,
1067                 'like_count': int,
1068                 'availability': 'public',
1069                 'playable_in_embed': True,
1070                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1071                 'live_status': 'not_live',
1072                 'age_limit': 0,
1073                 'start_time': 1,
1074                 'end_time': 9,
1075                 'channel_follower_count': int
1076             }
1077         },
1078         {
1079             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1080             'note': 'Embed-only video (#1746)',
1081             'info_dict': {
1082                 'id': 'yZIXLfi8CZQ',
1083                 'ext': 'mp4',
1084                 'upload_date': '20120608',
1085                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1086                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1087                 'uploader': 'SET India',
1088                 'uploader_id': 'setindia',
1089                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1090                 'age_limit': 18,
1091             },
1092             'skip': 'Private video',
1093         },
1094         {
1095             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1096             'note': 'Use the first video ID in the URL',
1097             'info_dict': {
1098                 'id': 'BaW_jenozKc',
1099                 'ext': 'mp4',
1100                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1101                 'uploader': 'Philipp Hagemeister',
1102                 'uploader_id': 'phihag',
1103                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1104                 'channel': 'Philipp Hagemeister',
1105                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1106                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1107                 'upload_date': '20121002',
1108                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1109                 'categories': ['Science & Technology'],
1110                 'tags': ['youtube-dl'],
1111                 'duration': 10,
1112                 'view_count': int,
1113                 'like_count': int,
1114                 'availability': 'public',
1115                 'playable_in_embed': True,
1116                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1117                 'live_status': 'not_live',
1118                 'age_limit': 0,
1119                 'channel_follower_count': int
1120             },
1121             'params': {
1122                 'skip_download': True,
1123             },
1124         },
1125         {
1126             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1127             'note': '256k DASH audio (format 141) via DASH manifest',
1128             'info_dict': {
1129                 'id': 'a9LDPn-MO4I',
1130                 'ext': 'm4a',
1131                 'upload_date': '20121002',
1132                 'uploader_id': '8KVIDEO',
1133                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1134                 'description': '',
1135                 'uploader': '8KVIDEO',
1136                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1137             },
1138             'params': {
1139                 'youtube_include_dash_manifest': True,
1140                 'format': '141',
1141             },
1142             'skip': 'format 141 not served anymore',
1143         },
1144         # DASH manifest with encrypted signature
1145         {
1146             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1147             'info_dict': {
1148                 'id': 'IB3lcPjvWLA',
1149                 'ext': 'm4a',
1150                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1151                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1152                 'duration': 244,
1153                 'uploader': 'AfrojackVEVO',
1154                 'uploader_id': 'AfrojackVEVO',
1155                 'upload_date': '20131011',
1156                 'abr': 129.495,
1157                 'like_count': int,
1158                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1159                 'playable_in_embed': True,
1160                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1161                 'view_count': int,
1162                 'track': 'The Spark',
1163                 'live_status': 'not_live',
1164                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1165                 'channel': 'Afrojack',
1166                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1167                 'tags': 'count:19',
1168                 'availability': 'public',
1169                 'categories': ['Music'],
1170                 'age_limit': 0,
1171                 'alt_title': 'The Spark',
1172                 'channel_follower_count': int
1173             },
1174             'params': {
1175                 'youtube_include_dash_manifest': True,
1176                 'format': '141/bestaudio[ext=m4a]',
1177             },
1178         },
1179         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1180         {
1181             'note': 'Embed allowed age-gate video',
1182             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1183             'info_dict': {
1184                 'id': 'HtVdAasjOgU',
1185                 'ext': 'mp4',
1186                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1187                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1188                 'duration': 142,
1189                 'uploader': 'The Witcher',
1190                 'uploader_id': 'WitcherGame',
1191                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1192                 'upload_date': '20140605',
1193                 'age_limit': 18,
1194                 'categories': ['Gaming'],
1195                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1196                 'availability': 'needs_auth',
1197                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1198                 'like_count': int,
1199                 'channel': 'The Witcher',
1200                 'live_status': 'not_live',
1201                 'tags': 'count:17',
1202                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1203                 'playable_in_embed': True,
1204                 'view_count': int,
1205                 'channel_follower_count': int
1206             },
1207         },
1208         {
1209             'note': 'Age-gate video with embed allowed in public site',
1210             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1211             'info_dict': {
1212                 'id': 'HsUATh_Nc2U',
1213                 'ext': 'mp4',
1214                 'title': 'Godzilla 2 (Official Video)',
1215                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1216                 'upload_date': '20200408',
1217                 'uploader_id': 'FlyingKitty900',
1218                 'uploader': 'FlyingKitty',
1219                 'age_limit': 18,
1220                 'availability': 'needs_auth',
1221                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1222                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1223                 'channel': 'FlyingKitty',
1224                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1225                 'view_count': int,
1226                 'categories': ['Entertainment'],
1227                 'live_status': 'not_live',
1228                 'tags': ['Flyingkitty', 'godzilla 2'],
1229                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1230                 'like_count': int,
1231                 'duration': 177,
1232                 'playable_in_embed': True,
1233                 'channel_follower_count': int
1234             },
1235         },
1236         {
1237             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1238             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1239             'info_dict': {
1240                 'id': 'Tq92D6wQ1mg',
1241                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1242                 'ext': 'mp4',
1243                 'upload_date': '20191228',
1244                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1245                 'uploader': 'Projekt Melody',
1246                 'description': 'md5:17eccca93a786d51bc67646756894066',
1247                 'age_limit': 18,
1248                 'like_count': int,
1249                 'availability': 'needs_auth',
1250                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1251                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1252                 'view_count': int,
1253                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1254                 'channel': 'Projekt Melody',
1255                 'live_status': 'not_live',
1256                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1257                 'playable_in_embed': True,
1258                 'categories': ['Entertainment'],
1259                 'duration': 106,
1260                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1261                 'channel_follower_count': int
1262             },
1263         },
1264         {
1265             'note': 'Non-Agegated non-embeddable video',
1266             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1267             'info_dict': {
1268                 'id': 'MeJVWBSsPAY',
1269                 'ext': 'mp4',
1270                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1271                 'uploader': 'Herr Lurik',
1272                 'uploader_id': 'st3in234',
1273                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1274                 'upload_date': '20130730',
1275                 'track': 'Such mich find mich',
1276                 'age_limit': 0,
1277                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1278                 'like_count': int,
1279                 'playable_in_embed': False,
1280                 'creator': 'OOMPH!',
1281                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1282                 'view_count': int,
1283                 'alt_title': 'Such mich find mich',
1284                 'duration': 210,
1285                 'channel': 'Herr Lurik',
1286                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1287                 'categories': ['Music'],
1288                 'availability': 'public',
1289                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1290                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1291                 'live_status': 'not_live',
1292                 'artist': 'OOMPH!',
1293                 'channel_follower_count': int
1294             },
1295         },
1296         {
1297             'note': 'Non-bypassable age-gated video',
1298             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1299             'only_matching': True,
1300         },
1301         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1302         # YouTube Red ad is not captured for creator
1303         {
1304             'url': '__2ABJjxzNo',
1305             'info_dict': {
1306                 'id': '__2ABJjxzNo',
1307                 'ext': 'mp4',
1308                 'duration': 266,
1309                 'upload_date': '20100430',
1310                 'uploader_id': 'deadmau5',
1311                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1312                 'creator': 'deadmau5',
1313                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1314                 'uploader': 'deadmau5',
1315                 'title': 'Deadmau5 - Some Chords (HD)',
1316                 'alt_title': 'Some Chords',
1317                 'availability': 'public',
1318                 'tags': 'count:14',
1319                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1320                 'view_count': int,
1321                 'live_status': 'not_live',
1322                 'channel': 'deadmau5',
1323                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1324                 'like_count': int,
1325                 'track': 'Some Chords',
1326                 'artist': 'deadmau5',
1327                 'playable_in_embed': True,
1328                 'age_limit': 0,
1329                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1330                 'categories': ['Music'],
1331                 'album': 'Some Chords',
1332                 'channel_follower_count': int
1333             },
1334             'expected_warnings': [
1335                 'DASH manifest missing',
1336             ]
1337         },
1338         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1339         {
1340             'url': 'lqQg6PlCWgI',
1341             'info_dict': {
1342                 'id': 'lqQg6PlCWgI',
1343                 'ext': 'mp4',
1344                 'duration': 6085,
1345                 'upload_date': '20150827',
1346                 'uploader_id': 'olympic',
1347                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1348                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1349                 'uploader': 'Olympics',
1350                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1351                 'like_count': int,
1352                 'release_timestamp': 1343767800,
1353                 'playable_in_embed': True,
1354                 'categories': ['Sports'],
1355                 'release_date': '20120731',
1356                 'channel': 'Olympics',
1357                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1358                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1359                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1360                 'age_limit': 0,
1361                 'availability': 'public',
1362                 'live_status': 'was_live',
1363                 'view_count': int,
1364                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1365                 'channel_follower_count': int
1366             },
1367             'params': {
1368                 'skip_download': 'requires avconv',
1369             }
1370         },
1371         # Non-square pixels
1372         {
1373             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1374             'info_dict': {
1375                 'id': '_b-2C3KPAM0',
1376                 'ext': 'mp4',
1377                 'stretched_ratio': 16 / 9.,
1378                 'duration': 85,
1379                 'upload_date': '20110310',
1380                 'uploader_id': 'AllenMeow',
1381                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1382                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1383                 'uploader': '孫ᄋᄅ',
1384                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1385                 'playable_in_embed': True,
1386                 'channel': '孫ᄋᄅ',
1387                 'age_limit': 0,
1388                 'tags': 'count:11',
1389                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1390                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1391                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1392                 'view_count': int,
1393                 'categories': ['People & Blogs'],
1394                 'like_count': int,
1395                 'live_status': 'not_live',
1396                 'availability': 'unlisted',
1397                 'channel_follower_count': int
1398             },
1399         },
1400         # url_encoded_fmt_stream_map is empty string
1401         {
1402             'url': 'qEJwOuvDf7I',
1403             'info_dict': {
1404                 'id': 'qEJwOuvDf7I',
1405                 'ext': 'webm',
1406                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1407                 'description': '',
1408                 'upload_date': '20150404',
1409                 'uploader_id': 'spbelect',
1410                 'uploader': 'Наблюдатели Петербурга',
1411             },
1412             'params': {
1413                 'skip_download': 'requires avconv',
1414             },
1415             'skip': 'This live event has ended.',
1416         },
1417         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1418         {
1419             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1420             'info_dict': {
1421                 'id': 'FIl7x6_3R5Y',
1422                 'ext': 'webm',
1423                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1424                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1425                 'duration': 220,
1426                 'upload_date': '20150625',
1427                 'uploader_id': 'dorappi2000',
1428                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1429                 'uploader': 'dorappi2000',
1430                 'formats': 'mincount:31',
1431             },
1432             'skip': 'not actual anymore',
1433         },
1434         # DASH manifest with segment_list
1435         {
1436             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1437             'md5': '8ce563a1d667b599d21064e982ab9e31',
1438             'info_dict': {
1439                 'id': 'CsmdDsKjzN8',
1440                 'ext': 'mp4',
1441                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1442                 'uploader': 'Airtek',
1443                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1444                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1445                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1446             },
1447             'params': {
1448                 'youtube_include_dash_manifest': True,
1449                 'format': '135',  # bestvideo
1450             },
1451             'skip': 'This live event has ended.',
1452         },
1453         {
1454             # Multifeed videos (multiple cameras), URL is for Main Camera
1455             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1456             'info_dict': {
1457                 'id': 'jvGDaLqkpTg',
1458                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1459                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1460             },
1461             'playlist': [{
1462                 'info_dict': {
1463                     'id': 'jvGDaLqkpTg',
1464                     'ext': 'mp4',
1465                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1466                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1467                     'duration': 10643,
1468                     'upload_date': '20161111',
1469                     'uploader': 'Team PGP',
1470                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1471                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1472                 },
1473             }, {
1474                 'info_dict': {
1475                     'id': '3AKt1R1aDnw',
1476                     'ext': 'mp4',
1477                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1478                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1479                     'duration': 10991,
1480                     'upload_date': '20161111',
1481                     'uploader': 'Team PGP',
1482                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1483                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1484                 },
1485             }, {
1486                 'info_dict': {
1487                     'id': 'RtAMM00gpVc',
1488                     'ext': 'mp4',
1489                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1490                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1491                     'duration': 10995,
1492                     'upload_date': '20161111',
1493                     'uploader': 'Team PGP',
1494                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1495                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1496                 },
1497             }, {
1498                 'info_dict': {
1499                     'id': '6N2fdlP3C5U',
1500                     'ext': 'mp4',
1501                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1502                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1503                     'duration': 10990,
1504                     'upload_date': '20161111',
1505                     'uploader': 'Team PGP',
1506                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1507                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1508                 },
1509             }],
1510             'params': {
1511                 'skip_download': True,
1512             },
1513             'skip': 'Not multifeed anymore',
1514         },
1515         {
1516             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1517             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1518             'info_dict': {
1519                 'id': 'gVfLd0zydlo',
1520                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1521             },
1522             'playlist_count': 2,
1523             'skip': 'Not multifeed anymore',
1524         },
1525         {
1526             'url': 'https://vid.plus/FlRa-iH7PGw',
1527             'only_matching': True,
1528         },
1529         {
1530             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1531             'only_matching': True,
1532         },
1533         {
1534             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1535             # Also tests cut-off URL expansion in video description (see
1536             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1537             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1538             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1539             'info_dict': {
1540                 'id': 'lsguqyKfVQg',
1541                 'ext': 'mp4',
1542                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1543                 'alt_title': 'Dark Walk',
1544                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1545                 'duration': 133,
1546                 'upload_date': '20151119',
1547                 'uploader_id': 'IronSoulElf',
1548                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1549                 'uploader': 'IronSoulElf',
1550                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1551                 'track': 'Dark Walk',
1552                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1553                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1554                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1555                 'categories': ['Film & Animation'],
1556                 'view_count': int,
1557                 'live_status': 'not_live',
1558                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1559                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1560                 'tags': 'count:13',
1561                 'availability': 'public',
1562                 'channel': 'IronSoulElf',
1563                 'playable_in_embed': True,
1564                 'like_count': int,
1565                 'age_limit': 0,
1566                 'channel_follower_count': int
1567             },
1568             'params': {
1569                 'skip_download': True,
1570             },
1571         },
1572         {
1573             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1574             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1575             'only_matching': True,
1576         },
1577         {
1578             # Video with yt:stretch=17:0
1579             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1580             'info_dict': {
1581                 'id': 'Q39EVAstoRM',
1582                 'ext': 'mp4',
1583                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1584                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1585                 'upload_date': '20151107',
1586                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1587                 'uploader': 'CH GAMER DROID',
1588             },
1589             'params': {
1590                 'skip_download': True,
1591             },
1592             'skip': 'This video does not exist.',
1593         },
1594         {
1595             # Video with incomplete 'yt:stretch=16:'
1596             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1597             'only_matching': True,
1598         },
1599         {
1600             # Video licensed under Creative Commons
1601             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1602             'info_dict': {
1603                 'id': 'M4gD1WSo5mA',
1604                 'ext': 'mp4',
1605                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1606                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1607                 'duration': 721,
1608                 'upload_date': '20150128',
1609                 'uploader_id': 'BerkmanCenter',
1610                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1611                 'uploader': 'The Berkman Klein Center for Internet & Society',
1612                 'license': 'Creative Commons Attribution license (reuse allowed)',
1613                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1614                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1615                 'like_count': int,
1616                 'age_limit': 0,
1617                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1618                 'channel': 'The Berkman Klein Center for Internet & Society',
1619                 'availability': 'public',
1620                 'view_count': int,
1621                 'categories': ['Education'],
1622                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1623                 'live_status': 'not_live',
1624                 'playable_in_embed': True,
1625                 'channel_follower_count': int
1626             },
1627             'params': {
1628                 'skip_download': True,
1629             },
1630         },
1631         {
1632             # Channel-like uploader_url
1633             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1634             'info_dict': {
1635                 'id': 'eQcmzGIKrzg',
1636                 'ext': 'mp4',
1637                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1638                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1639                 'duration': 4060,
1640                 'upload_date': '20151120',
1641                 'uploader': 'Bernie Sanders',
1642                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1643                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1644                 'license': 'Creative Commons Attribution license (reuse allowed)',
1645                 'playable_in_embed': True,
1646                 'tags': 'count:12',
1647                 'like_count': int,
1648                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1649                 'age_limit': 0,
1650                 'availability': 'public',
1651                 'categories': ['News & Politics'],
1652                 'channel': 'Bernie Sanders',
1653                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1654                 'view_count': int,
1655                 'live_status': 'not_live',
1656                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1657                 'channel_follower_count': int
1658             },
1659             'params': {
1660                 'skip_download': True,
1661             },
1662         },
1663         {
1664             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1665             'only_matching': True,
1666         },
1667         {
1668             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1669             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1670             'only_matching': True,
1671         },
1672         {
1673             # Rental video preview
1674             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1675             'info_dict': {
1676                 'id': 'uGpuVWrhIzE',
1677                 'ext': 'mp4',
1678                 'title': 'Piku - Trailer',
1679                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1680                 'upload_date': '20150811',
1681                 'uploader': 'FlixMatrix',
1682                 'uploader_id': 'FlixMatrixKaravan',
1683                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1684                 'license': 'Standard YouTube License',
1685             },
1686             'params': {
1687                 'skip_download': True,
1688             },
1689             'skip': 'This video is not available.',
1690         },
1691         {
1692             # YouTube Red video with episode data
1693             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1694             'info_dict': {
1695                 'id': 'iqKdEhx-dD4',
1696                 'ext': 'mp4',
1697                 'title': 'Isolation - Mind Field (Ep 1)',
1698                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1699                 'duration': 2085,
1700                 'upload_date': '20170118',
1701                 'uploader': 'Vsauce',
1702                 'uploader_id': 'Vsauce',
1703                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1704                 'series': 'Mind Field',
1705                 'season_number': 1,
1706                 'episode_number': 1,
1707                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1708                 'tags': 'count:12',
1709                 'view_count': int,
1710                 'availability': 'public',
1711                 'age_limit': 0,
1712                 'channel': 'Vsauce',
1713                 'episode': 'Episode 1',
1714                 'categories': ['Entertainment'],
1715                 'season': 'Season 1',
1716                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1717                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1718                 'like_count': int,
1719                 'playable_in_embed': True,
1720                 'live_status': 'not_live',
1721                 'channel_follower_count': int
1722             },
1723             'params': {
1724                 'skip_download': True,
1725             },
1726             'expected_warnings': [
1727                 'Skipping DASH manifest',
1728             ],
1729         },
1730         {
1731             # The following content has been identified by the YouTube community
1732             # as inappropriate or offensive to some audiences.
1733             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1734             'info_dict': {
1735                 'id': '6SJNVb0GnPI',
1736                 'ext': 'mp4',
1737                 'title': 'Race Differences in Intelligence',
1738                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1739                 'duration': 965,
1740                 'upload_date': '20140124',
1741                 'uploader': 'New Century Foundation',
1742                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1743                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1744             },
1745             'params': {
1746                 'skip_download': True,
1747             },
1748             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1749         },
1750         {
1751             # itag 212
1752             'url': '1t24XAntNCY',
1753             'only_matching': True,
1754         },
1755         {
1756             # geo restricted to JP
1757             'url': 'sJL6WA-aGkQ',
1758             'only_matching': True,
1759         },
1760         {
1761             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1762             'only_matching': True,
1763         },
1764         {
1765             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1766             'only_matching': True,
1767         },
1768         {
1769             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1770             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1771             'only_matching': True,
1772         },
1773         {
1774             # DRM protected
1775             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1776             'only_matching': True,
1777         },
1778         {
1779             # Video with unsupported adaptive stream type formats
1780             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1781             'info_dict': {
1782                 'id': 'Z4Vy8R84T1U',
1783                 'ext': 'mp4',
1784                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1785                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1786                 'duration': 433,
1787                 'upload_date': '20130923',
1788                 'uploader': 'Amelia Putri Harwita',
1789                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1790                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1791                 'formats': 'maxcount:10',
1792             },
1793             'params': {
1794                 'skip_download': True,
1795                 'youtube_include_dash_manifest': False,
1796             },
1797             'skip': 'not actual anymore',
1798         },
1799         {
1800             # Youtube Music Auto-generated description
1801             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1802             'info_dict': {
1803                 'id': 'MgNrAu2pzNs',
1804                 'ext': 'mp4',
1805                 'title': 'Voyeur Girl',
1806                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1807                 'upload_date': '20190312',
1808                 'uploader': 'Stephen - Topic',
1809                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1810                 'artist': 'Stephen',
1811                 'track': 'Voyeur Girl',
1812                 'album': 'it\'s too much love to know my dear',
1813                 'release_date': '20190313',
1814                 'release_year': 2019,
1815                 'alt_title': 'Voyeur Girl',
1816                 'view_count': int,
1817                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1818                 'playable_in_embed': True,
1819                 'like_count': int,
1820                 'categories': ['Music'],
1821                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1822                 'channel': 'Stephen',
1823                 'availability': 'public',
1824                 'creator': 'Stephen',
1825                 'duration': 169,
1826                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1827                 'age_limit': 0,
1828                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1829                 'tags': 'count:11',
1830                 'live_status': 'not_live',
1831                 'channel_follower_count': int
1832             },
1833             'params': {
1834                 'skip_download': True,
1835             },
1836         },
1837         {
1838             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1839             'only_matching': True,
1840         },
1841         {
1842             # invalid -> valid video id redirection
1843             'url': 'DJztXj2GPfl',
1844             'info_dict': {
1845                 'id': 'DJztXj2GPfk',
1846                 'ext': 'mp4',
1847                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1848                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1849                 'upload_date': '20090125',
1850                 'uploader': 'Prochorowka',
1851                 'uploader_id': 'Prochorowka',
1852                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1853                 'artist': 'Panjabi MC',
1854                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1855                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1856             },
1857             'params': {
1858                 'skip_download': True,
1859             },
1860             'skip': 'Video unavailable',
1861         },
1862         {
1863             # empty description results in an empty string
1864             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1865             'info_dict': {
1866                 'id': 'x41yOUIvK2k',
1867                 'ext': 'mp4',
1868                 'title': 'IMG 3456',
1869                 'description': '',
1870                 'upload_date': '20170613',
1871                 'uploader_id': 'ElevageOrVert',
1872                 'uploader': 'ElevageOrVert',
1873                 'view_count': int,
1874                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1875                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1876                 'like_count': int,
1877                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1878                 'tags': [],
1879                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1880                 'availability': 'public',
1881                 'age_limit': 0,
1882                 'categories': ['Pets & Animals'],
1883                 'duration': 7,
1884                 'playable_in_embed': True,
1885                 'live_status': 'not_live',
1886                 'channel': 'ElevageOrVert',
1887                 'channel_follower_count': int
1888             },
1889             'params': {
1890                 'skip_download': True,
1891             },
1892         },
1893         {
1894             # with '};' inside yt initial data (see [1])
1895             # see [2] for an example with '};' inside ytInitialPlayerResponse
1896             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1897             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1898             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1899             'info_dict': {
1900                 'id': 'CHqg6qOn4no',
1901                 'ext': 'mp4',
1902                 'title': 'Part 77   Sort a list of simple types in c#',
1903                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1904                 'upload_date': '20130831',
1905                 'uploader_id': 'kudvenkat',
1906                 'uploader': 'kudvenkat',
1907                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1908                 'like_count': int,
1909                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1910                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1911                 'live_status': 'not_live',
1912                 'categories': ['Education'],
1913                 'availability': 'public',
1914                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1915                 'tags': 'count:12',
1916                 'playable_in_embed': True,
1917                 'age_limit': 0,
1918                 'view_count': int,
1919                 'duration': 522,
1920                 'channel': 'kudvenkat',
1921                 'channel_follower_count': int
1922             },
1923             'params': {
1924                 'skip_download': True,
1925             },
1926         },
1927         {
1928             # another example of '};' in ytInitialData
1929             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1930             'only_matching': True,
1931         },
1932         {
1933             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1934             'only_matching': True,
1935         },
1936         {
1937             # https://github.com/ytdl-org/youtube-dl/pull/28094
1938             'url': 'OtqTfy26tG0',
1939             'info_dict': {
1940                 'id': 'OtqTfy26tG0',
1941                 'ext': 'mp4',
1942                 'title': 'Burn Out',
1943                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1944                 'upload_date': '20141120',
1945                 'uploader': 'The Cinematic Orchestra - Topic',
1946                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1947                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1948                 'artist': 'The Cinematic Orchestra',
1949                 'track': 'Burn Out',
1950                 'album': 'Every Day',
1951                 'like_count': int,
1952                 'live_status': 'not_live',
1953                 'alt_title': 'Burn Out',
1954                 'duration': 614,
1955                 'age_limit': 0,
1956                 'view_count': int,
1957                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1958                 'creator': 'The Cinematic Orchestra',
1959                 'channel': 'The Cinematic Orchestra',
1960                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1961                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1962                 'availability': 'public',
1963                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1964                 'categories': ['Music'],
1965                 'playable_in_embed': True,
1966                 'channel_follower_count': int
1967             },
1968             'params': {
1969                 'skip_download': True,
1970             },
1971         },
1972         {
1973             # controversial video, only works with bpctr when authenticated with cookies
1974             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1975             'only_matching': True,
1976         },
1977         {
1978             # controversial video, requires bpctr/contentCheckOk
1979             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1980             'info_dict': {
1981                 'id': 'SZJvDhaSDnc',
1982                 'ext': 'mp4',
1983                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1984                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1985                 'uploader': 'CBS Mornings',
1986                 'uploader_id': 'CBSThisMorning',
1987                 'upload_date': '20140716',
1988                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1989                 'duration': 170,
1990                 'categories': ['News & Politics'],
1991                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
1992                 'view_count': int,
1993                 'channel': 'CBS Mornings',
1994                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
1995                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
1996                 'age_limit': 18,
1997                 'availability': 'needs_auth',
1998                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
1999                 'like_count': int,
2000                 'live_status': 'not_live',
2001                 'playable_in_embed': True,
2002                 'channel_follower_count': int
2003             }
2004         },
2005         {
2006             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2007             'url': 'cBvYw8_A0vQ',
2008             'info_dict': {
2009                 'id': 'cBvYw8_A0vQ',
2010                 'ext': 'mp4',
2011                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2012                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2013                 'upload_date': '20201120',
2014                 'uploader': 'Walk around Japan',
2015                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2016                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2017                 'duration': 1456,
2018                 'categories': ['Travel & Events'],
2019                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2020                 'view_count': int,
2021                 'channel': 'Walk around Japan',
2022                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2023                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2024                 'age_limit': 0,
2025                 'availability': 'public',
2026                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2027                 'live_status': 'not_live',
2028                 'playable_in_embed': True,
2029                 'channel_follower_count': int
2030             },
2031             'params': {
2032                 'skip_download': True,
2033             },
2034         }, {
2035             # Has multiple audio streams
2036             'url': 'WaOKSUlf4TM',
2037             'only_matching': True
2038         }, {
2039             # Requires Premium: has format 141 when requested using YTM url
2040             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2041             'only_matching': True
2042         }, {
2043             # multiple subtitles with same lang_code
2044             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2045             'only_matching': True,
2046         }, {
2047             # Force use android client fallback
2048             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2049             'info_dict': {
2050                 'id': 'YOelRv7fMxY',
2051                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2052                 'ext': '3gp',
2053                 'upload_date': '20210624',
2054                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2055                 'uploader': 'colinfurze',
2056                 'uploader_id': 'colinfurze',
2057                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2058                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2059                 'duration': 596,
2060                 'categories': ['Entertainment'],
2061                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2062                 'view_count': int,
2063                 'channel': 'colinfurze',
2064                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2065                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2066                 'age_limit': 0,
2067                 'availability': 'public',
2068                 'like_count': int,
2069                 'live_status': 'not_live',
2070                 'playable_in_embed': True,
2071                 'channel_follower_count': int
2072             },
2073             'params': {
2074                 'format': '17',  # 3gp format available on android
2075                 'extractor_args': {'youtube': {'player_client': ['android']}},
2076             },
2077         },
2078         {
2079             # Skip download of additional client configs (remix client config in this case)
2080             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2081             'only_matching': True,
2082             'params': {
2083                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2084             },
2085         }, {
2086             # shorts
2087             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2088             'only_matching': True,
2089         }, {
2090             'note': 'Storyboards',
2091             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2092             'info_dict': {
2093                 'id': '5KLPxDtMqe8',
2094                 'ext': 'mhtml',
2095                 'format_id': 'sb0',
2096                 'title': 'Your Brain is Plastic',
2097                 'uploader_id': 'scishow',
2098                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2099                 'upload_date': '20140324',
2100                 'uploader': 'SciShow',
2101                 'like_count': int,
2102                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2103                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2104                 'view_count': int,
2105                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2106                 'playable_in_embed': True,
2107                 'tags': 'count:12',
2108                 'uploader_url': 'http://www.youtube.com/user/scishow',
2109                 'availability': 'public',
2110                 'channel': 'SciShow',
2111                 'live_status': 'not_live',
2112                 'duration': 248,
2113                 'categories': ['Education'],
2114                 'age_limit': 0,
2115                 'channel_follower_count': int
2116             }, 'params': {'format': 'mhtml', 'skip_download': True}
2117         }, {
2118             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2119             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2120             'info_dict': {
2121                 'id': '2NUZ8W2llS4',
2122                 'ext': 'mp4',
2123                 'title': 'The NP that test your phone performance 🙂',
2124                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2125                 'uploader': 'Leon Nguyen',
2126                 'uploader_id': 'VNSXIII',
2127                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2128                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2129                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2130                 'duration': 21,
2131                 'view_count': int,
2132                 'age_limit': 0,
2133                 'categories': ['Gaming'],
2134                 'tags': 'count:23',
2135                 'playable_in_embed': True,
2136                 'live_status': 'not_live',
2137                 'upload_date': '20220103',
2138                 'like_count': int,
2139                 'availability': 'public',
2140                 'channel': 'Leon Nguyen',
2141                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2142                 'channel_follower_count': int
2143             }
2144         }, {
2145             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2146             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2147             'info_dict': {
2148                 'id': 'mzZzzBU6lrM',
2149                 'ext': 'mp4',
2150                 'title': 'I Met GeorgeNotFound In Real Life...',
2151                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2152                 'uploader': 'Quackity',
2153                 'uploader_id': 'QuackityHQ',
2154                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2155                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2156                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2157                 'duration': 955,
2158                 'view_count': int,
2159                 'age_limit': 0,
2160                 'categories': ['Entertainment'],
2161                 'tags': 'count:26',
2162                 'playable_in_embed': True,
2163                 'live_status': 'not_live',
2164                 'release_timestamp': 1641172509,
2165                 'release_date': '20220103',
2166                 'upload_date': '20220103',
2167                 'like_count': int,
2168                 'availability': 'public',
2169                 'channel': 'Quackity',
2170                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2171                 'channel_follower_count': int
2172             }
2173         },
2174         {   # continuous livestream. Microformat upload date should be preferred.
2175             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2176             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2177             'info_dict': {
2178                 'id': 'kgx4WGK0oNU',
2179                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2180                 'ext': 'mp4',
2181                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2182                 'availability': 'public',
2183                 'age_limit': 0,
2184                 'release_timestamp': 1637975704,
2185                 'upload_date': '20210619',
2186                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2187                 'live_status': 'is_live',
2188                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2189                 'uploader': '阿鲍Abao',
2190                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2191                 'channel': 'Abao in Tokyo',
2192                 'channel_follower_count': int,
2193                 'release_date': '20211127',
2194                 'tags': 'count:39',
2195                 'categories': ['People & Blogs'],
2196                 'like_count': int,
2197                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2198                 'view_count': int,
2199                 'playable_in_embed': True,
2200                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2201             },
2202             'params': {'skip_download': True}
2203         }, {
2204             # Story. Requires specific player params to work.
2205             # Note: stories get removed after some period of time
2206             'url': 'https://www.youtube.com/watch?v=yN3x1t3sieA',
2207             'info_dict': {
2208                 'id': 'yN3x1t3sieA',
2209                 'ext': 'mp4',
2210                 'uploader': 'Linus Tech Tips',
2211                 'duration': 13,
2212                 'channel': 'Linus Tech Tips',
2213                 'playable_in_embed': True,
2214                 'tags': [],
2215                 'age_limit': 0,
2216                 'uploader_url': 'http://www.youtube.com/user/LinusTechTips',
2217                 'upload_date': '20220402',
2218                 'thumbnail': 'https://i.ytimg.com/vi_webp/yN3x1t3sieA/maxresdefault.webp',
2219                 'title': 'Story',
2220                 'live_status': 'not_live',
2221                 'uploader_id': 'LinusTechTips',
2222                 'view_count': int,
2223                 'description': '',
2224                 'channel_id': 'UCXuqSBlHAE6Xw-yeJA0Tunw',
2225                 'categories': ['Science & Technology'],
2226                 'channel_url': 'https://www.youtube.com/channel/UCXuqSBlHAE6Xw-yeJA0Tunw',
2227                 'availability': 'unlisted',
2228             }
2229         }
2230     ]
2231
2232     @classmethod
2233     def suitable(cls, url):
2234         from ..utils import parse_qs
2235
2236         qs = parse_qs(url)
2237         if qs.get('list', [None])[0]:
2238             return False
2239         return super().suitable(url)
2240
2241     def __init__(self, *args, **kwargs):
2242         super().__init__(*args, **kwargs)
2243         self._code_cache = {}
2244         self._player_cache = {}
2245
2246     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2247         lock = threading.Lock()
2248
2249         is_live = True
2250         start_time = time.time()
2251         formats = [f for f in formats if f.get('is_from_start')]
2252
2253         def refetch_manifest(format_id, delay):
2254             nonlocal formats, start_time, is_live
2255             if time.time() <= start_time + delay:
2256                 return
2257
2258             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2259             video_details = traverse_obj(
2260                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2261             microformats = traverse_obj(
2262                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2263                 expected_type=dict, default=[])
2264             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2265             start_time = time.time()
2266
2267         def mpd_feed(format_id, delay):
2268             """
2269             @returns (manifest_url, manifest_stream_number, is_live) or None
2270             """
2271             with lock:
2272                 refetch_manifest(format_id, delay)
2273
2274             f = next((f for f in formats if f['format_id'] == format_id), None)
2275             if not f:
2276                 if not is_live:
2277                     self.to_screen(f'{video_id}: Video is no longer live')
2278                 else:
2279                     self.report_warning(
2280                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2281                 return None
2282             return f['manifest_url'], f['manifest_stream_number'], is_live
2283
2284         for f in formats:
2285             f['is_live'] = True
2286             f['protocol'] = 'http_dash_segments_generator'
2287             f['fragments'] = functools.partial(
2288                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2289
2290     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2291         FETCH_SPAN, MAX_DURATION = 5, 432000
2292
2293         mpd_url, stream_number, is_live = None, None, True
2294
2295         begin_index = 0
2296         download_start_time = ctx.get('start') or time.time()
2297
2298         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2299         if lack_early_segments:
2300             self.report_warning(bug_reports_message(
2301                 'Starting download from the last 120 hours of the live stream since '
2302                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2303             lack_early_segments = True
2304
2305         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2306         fragments, fragment_base_url = None, None
2307
2308         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2309             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2310             # Obtain from MPD's maximum seq value
2311             old_mpd_url = mpd_url
2312             last_error = ctx.pop('last_error', None)
2313             expire_fast = immediate or last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2314             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2315                                                or (mpd_url, stream_number, False))
2316             if not refresh_sequence:
2317                 if expire_fast and not is_live:
2318                     return False, last_seq
2319                 elif old_mpd_url == mpd_url:
2320                     return True, last_seq
2321             try:
2322                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2323                     mpd_url, None, note=False, errnote=False, fatal=False)
2324             except ExtractorError:
2325                 fmts = None
2326             if not fmts:
2327                 no_fragment_score += 2
2328                 return False, last_seq
2329             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2330             fragments = fmt_info['fragments']
2331             fragment_base_url = fmt_info['fragment_base_url']
2332             assert fragment_base_url
2333
2334             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2335             return True, _last_seq
2336
2337         while is_live:
2338             fetch_time = time.time()
2339             if no_fragment_score > 30:
2340                 return
2341             if last_segment_url:
2342                 # Obtain from "X-Head-Seqnum" header value from each segment
2343                 try:
2344                     urlh = self._request_webpage(
2345                         last_segment_url, None, note=False, errnote=False, fatal=False)
2346                 except ExtractorError:
2347                     urlh = None
2348                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2349                 if last_seq is None:
2350                     no_fragment_score += 2
2351                     last_segment_url = None
2352                     continue
2353             else:
2354                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2355                 no_fragment_score += 2
2356                 if not should_continue:
2357                     continue
2358
2359             if known_idx > last_seq:
2360                 last_segment_url = None
2361                 continue
2362
2363             last_seq += 1
2364
2365             if begin_index < 0 and known_idx < 0:
2366                 # skip from the start when it's negative value
2367                 known_idx = last_seq + begin_index
2368             if lack_early_segments:
2369                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2370             try:
2371                 for idx in range(known_idx, last_seq):
2372                     # do not update sequence here or you'll get skipped some part of it
2373                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2374                     if not should_continue:
2375                         known_idx = idx - 1
2376                         raise ExtractorError('breaking out of outer loop')
2377                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2378                     yield {
2379                         'url': last_segment_url,
2380                     }
2381                 if known_idx == last_seq:
2382                     no_fragment_score += 5
2383                 else:
2384                     no_fragment_score = 0
2385                 known_idx = last_seq
2386             except ExtractorError:
2387                 continue
2388
2389             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2390
2391     def _extract_player_url(self, *ytcfgs, webpage=None):
2392         player_url = traverse_obj(
2393             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2394             get_all=False, expected_type=compat_str)
2395         if not player_url:
2396             return
2397         return urljoin('https://www.youtube.com', player_url)
2398
2399     def _download_player_url(self, video_id, fatal=False):
2400         res = self._download_webpage(
2401             'https://www.youtube.com/iframe_api',
2402             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2403         if res:
2404             player_version = self._search_regex(
2405                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2406             if player_version:
2407                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2408
2409     def _signature_cache_id(self, example_sig):
2410         """ Return a string representation of a signature """
2411         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2412
2413     @classmethod
2414     def _extract_player_info(cls, player_url):
2415         for player_re in cls._PLAYER_INFO_RE:
2416             id_m = re.search(player_re, player_url)
2417             if id_m:
2418                 break
2419         else:
2420             raise ExtractorError('Cannot identify player %r' % player_url)
2421         return id_m.group('id')
2422
2423     def _load_player(self, video_id, player_url, fatal=True):
2424         player_id = self._extract_player_info(player_url)
2425         if player_id not in self._code_cache:
2426             code = self._download_webpage(
2427                 player_url, video_id, fatal=fatal,
2428                 note='Downloading player ' + player_id,
2429                 errnote='Download of %s failed' % player_url)
2430             if code:
2431                 self._code_cache[player_id] = code
2432         return self._code_cache.get(player_id)
2433
2434     def _extract_signature_function(self, video_id, player_url, example_sig):
2435         player_id = self._extract_player_info(player_url)
2436
2437         # Read from filesystem cache
2438         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2439         assert os.path.basename(func_id) == func_id
2440
2441         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
2442         if cache_spec is not None:
2443             return lambda s: ''.join(s[i] for i in cache_spec)
2444
2445         code = self._load_player(video_id, player_url)
2446         if code:
2447             res = self._parse_sig_js(code)
2448
2449             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2450             cache_res = res(test_string)
2451             cache_spec = [ord(c) for c in cache_res]
2452
2453             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
2454             return res
2455
2456     def _print_sig_code(self, func, example_sig):
2457         if not self.get_param('youtube_print_sig_code'):
2458             return
2459
2460         def gen_sig_code(idxs):
2461             def _genslice(start, end, step):
2462                 starts = '' if start == 0 else str(start)
2463                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2464                 steps = '' if step == 1 else (':%d' % step)
2465                 return f's[{starts}{ends}{steps}]'
2466
2467             step = None
2468             # Quelch pyflakes warnings - start will be set when step is set
2469             start = '(Never used)'
2470             for i, prev in zip(idxs[1:], idxs[:-1]):
2471                 if step is not None:
2472                     if i - prev == step:
2473                         continue
2474                     yield _genslice(start, prev, step)
2475                     step = None
2476                     continue
2477                 if i - prev in [-1, 1]:
2478                     step = i - prev
2479                     start = prev
2480                     continue
2481                 else:
2482                     yield 's[%d]' % prev
2483             if step is None:
2484                 yield 's[%d]' % i
2485             else:
2486                 yield _genslice(start, i, step)
2487
2488         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2489         cache_res = func(test_string)
2490         cache_spec = [ord(c) for c in cache_res]
2491         expr_code = ' + '.join(gen_sig_code(cache_spec))
2492         signature_id_tuple = '(%s)' % (
2493             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2494         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2495                 '    return %s\n') % (signature_id_tuple, expr_code)
2496         self.to_screen('Extracted signature function:\n' + code)
2497
2498     def _parse_sig_js(self, jscode):
2499         funcname = self._search_regex(
2500             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2501              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2502              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2503              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2504              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2505              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2506              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2507              # Obsolete patterns
2508              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2509              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2510              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2511              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2512              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2513              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2514              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2515              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2516             jscode, 'Initial JS player signature function name', group='sig')
2517
2518         jsi = JSInterpreter(jscode)
2519         initial_function = jsi.extract_function(funcname)
2520         return lambda s: initial_function([s])
2521
2522     def _decrypt_signature(self, s, video_id, player_url):
2523         """Turn the encrypted s field into a working signature"""
2524
2525         if player_url is None:
2526             raise ExtractorError('Cannot decrypt signature without player_url')
2527
2528         try:
2529             player_id = (player_url, self._signature_cache_id(s))
2530             if player_id not in self._player_cache:
2531                 func = self._extract_signature_function(
2532                     video_id, player_url, s
2533                 )
2534                 self._player_cache[player_id] = func
2535             func = self._player_cache[player_id]
2536             self._print_sig_code(func, s)
2537             return func(s)
2538         except Exception as e:
2539             raise ExtractorError('Signature extraction failed: ' + traceback.format_exc(), cause=e)
2540
2541     def _decrypt_nsig(self, s, video_id, player_url):
2542         """Turn the encrypted n field into a working signature"""
2543         if player_url is None:
2544             raise ExtractorError('Cannot decrypt nsig without player_url')
2545         player_url = urljoin('https://www.youtube.com', player_url)
2546
2547         sig_id = ('nsig_value', s)
2548         if sig_id in self._player_cache:
2549             return self._player_cache[sig_id]
2550
2551         try:
2552             player_id = ('nsig', player_url)
2553             if player_id not in self._player_cache:
2554                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2555             func = self._player_cache[player_id]
2556             self._player_cache[sig_id] = func(s)
2557             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2558             return self._player_cache[sig_id]
2559         except Exception as e:
2560             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2561
2562     def _extract_n_function_name(self, jscode):
2563         nfunc, idx = self._search_regex(
2564             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2565             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2566         if not idx:
2567             return nfunc
2568         return json.loads(js_to_json(self._search_regex(
2569             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2570             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2571
2572     def _extract_n_function(self, video_id, player_url):
2573         player_id = self._extract_player_info(player_url)
2574         func_code = self._downloader.cache.load('youtube-nsig', player_id)
2575
2576         if func_code:
2577             jsi = JSInterpreter(func_code)
2578         else:
2579             jscode = self._load_player(video_id, player_url)
2580             funcname = self._extract_n_function_name(jscode)
2581             jsi = JSInterpreter(jscode)
2582             func_code = jsi.extract_function_code(funcname)
2583             self._downloader.cache.store('youtube-nsig', player_id, func_code)
2584
2585         if self.get_param('youtube_print_sig_code'):
2586             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2587
2588         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2589
2590     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2591         """
2592         Extract signatureTimestamp (sts)
2593         Required to tell API what sig/player version is in use.
2594         """
2595         sts = None
2596         if isinstance(ytcfg, dict):
2597             sts = int_or_none(ytcfg.get('STS'))
2598
2599         if not sts:
2600             # Attempt to extract from player
2601             if player_url is None:
2602                 error_msg = 'Cannot extract signature timestamp without player_url.'
2603                 if fatal:
2604                     raise ExtractorError(error_msg)
2605                 self.report_warning(error_msg)
2606                 return
2607             code = self._load_player(video_id, player_url, fatal=fatal)
2608             if code:
2609                 sts = int_or_none(self._search_regex(
2610                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2611                     'JS player signature timestamp', group='sts', fatal=fatal))
2612         return sts
2613
2614     def _mark_watched(self, video_id, player_responses):
2615         playback_url = get_first(
2616             player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2617             expected_type=url_or_none)
2618         if not playback_url:
2619             self.report_warning('Unable to mark watched')
2620             return
2621         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2622         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2623
2624         # cpn generation algorithm is reverse engineered from base.js.
2625         # In fact it works even with dummy cpn.
2626         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2627         cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2628
2629         qs.update({
2630             'ver': ['2'],
2631             'cpn': [cpn],
2632         })
2633         playback_url = compat_urlparse.urlunparse(
2634             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2635
2636         self._download_webpage(
2637             playback_url, video_id, 'Marking watched',
2638             'Unable to mark watched', fatal=False)
2639
2640     @staticmethod
2641     def _extract_urls(webpage):
2642         # Embedded YouTube player
2643         entries = [
2644             unescapeHTML(mobj.group('url'))
2645             for mobj in re.finditer(r'''(?x)
2646             (?:
2647                 <iframe[^>]+?src=|
2648                 data-video-url=|
2649                 <embed[^>]+?src=|
2650                 embedSWF\(?:\s*|
2651                 <object[^>]+data=|
2652                 new\s+SWFObject\(
2653             )
2654             (["\'])
2655                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2656                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2657             \1''', webpage)]
2658
2659         # lazyYT YouTube embed
2660         entries.extend(list(map(
2661             unescapeHTML,
2662             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2663
2664         # Wordpress "YouTube Video Importer" plugin
2665         matches = re.findall(r'''(?x)<div[^>]+
2666             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2667             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2668         entries.extend(m[-1] for m in matches)
2669
2670         return entries
2671
2672     @staticmethod
2673     def _extract_url(webpage):
2674         urls = YoutubeIE._extract_urls(webpage)
2675         return urls[0] if urls else None
2676
2677     @classmethod
2678     def extract_id(cls, url):
2679         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2680         if mobj is None:
2681             raise ExtractorError('Invalid URL: %s' % url)
2682         return mobj.group('id')
2683
2684     def _extract_chapters_from_json(self, data, duration):
2685         chapter_list = traverse_obj(
2686             data, (
2687                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2688                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2689             ), expected_type=list)
2690
2691         return self._extract_chapters(
2692             chapter_list,
2693             chapter_time=lambda chapter: float_or_none(
2694                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2695             chapter_title=lambda chapter: traverse_obj(
2696                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2697             duration=duration)
2698
2699     def _extract_chapters_from_engagement_panel(self, data, duration):
2700         content_list = traverse_obj(
2701             data,
2702             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2703             expected_type=list, default=[])
2704         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2705         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2706
2707         return next((
2708             filter(None, (
2709                 self._extract_chapters(
2710                     traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2711                     chapter_time, chapter_title, duration)
2712                 for contents in content_list
2713             ))), [])
2714
2715     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration):
2716         chapters = []
2717         last_chapter = {'start_time': 0}
2718         for idx, chapter in enumerate(chapter_list or []):
2719             title = chapter_title(chapter)
2720             start_time = chapter_time(chapter)
2721             if start_time is None:
2722                 continue
2723             last_chapter['end_time'] = start_time
2724             if start_time < last_chapter['start_time']:
2725                 if idx == 1:
2726                     chapters.pop()
2727                     self.report_warning('Invalid start time for chapter "%s"' % last_chapter['title'])
2728                 else:
2729                     self.report_warning(f'Invalid start time for chapter "{title}"')
2730                     continue
2731             last_chapter = {'start_time': start_time, 'title': title}
2732             chapters.append(last_chapter)
2733         last_chapter['end_time'] = duration
2734         return chapters
2735
2736     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
2737         return self._parse_json(self._search_regex(
2738             (fr'{regex}\s*{self._YT_INITIAL_BOUNDARY_RE}',
2739              regex), webpage, name, default='{}'), video_id, fatal=False)
2740
2741     def _extract_comment(self, comment_renderer, parent=None):
2742         comment_id = comment_renderer.get('commentId')
2743         if not comment_id:
2744             return
2745
2746         text = self._get_text(comment_renderer, 'contentText')
2747
2748         # note: timestamp is an estimate calculated from the current time and time_text
2749         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2750         author = self._get_text(comment_renderer, 'authorText')
2751         author_id = try_get(comment_renderer,
2752                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2753
2754         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2755                                                        lambda x: x['likeCount']), compat_str)) or 0
2756         author_thumbnail = try_get(comment_renderer,
2757                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2758
2759         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2760         is_favorited = 'creatorHeart' in (try_get(
2761             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2762         return {
2763             'id': comment_id,
2764             'text': text,
2765             'timestamp': timestamp,
2766             'time_text': time_text,
2767             'like_count': votes,
2768             'is_favorited': is_favorited,
2769             'author': author,
2770             'author_id': author_id,
2771             'author_thumbnail': author_thumbnail,
2772             'author_is_uploader': author_is_uploader,
2773             'parent': parent or 'root'
2774         }
2775
2776     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2777
2778         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2779
2780         def extract_header(contents):
2781             _continuation = None
2782             for content in contents:
2783                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2784                 expected_comment_count = self._get_count(
2785                     comments_header_renderer, 'countText', 'commentsCount')
2786
2787                 if expected_comment_count:
2788                     tracker['est_total'] = expected_comment_count
2789                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2790                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2791
2792                 sort_menu_item = try_get(
2793                     comments_header_renderer,
2794                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2795                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2796
2797                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2798                 if not _continuation:
2799                     continue
2800
2801                 sort_text = str_or_none(sort_menu_item.get('title'))
2802                 if not sort_text:
2803                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2804                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2805                 break
2806             return _continuation
2807
2808         def extract_thread(contents):
2809             if not parent:
2810                 tracker['current_page_thread'] = 0
2811             for content in contents:
2812                 if not parent and tracker['total_parent_comments'] >= max_parents:
2813                     yield
2814                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2815                 comment_renderer = get_first(
2816                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2817                     expected_type=dict, default={})
2818
2819                 comment = self._extract_comment(comment_renderer, parent)
2820                 if not comment:
2821                     continue
2822
2823                 tracker['running_total'] += 1
2824                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2825                 yield comment
2826
2827                 # Attempt to get the replies
2828                 comment_replies_renderer = try_get(
2829                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2830
2831                 if comment_replies_renderer:
2832                     tracker['current_page_thread'] += 1
2833                     comment_entries_iter = self._comment_entries(
2834                         comment_replies_renderer, ytcfg, video_id,
2835                         parent=comment.get('id'), tracker=tracker)
2836                     yield from itertools.islice(comment_entries_iter, min(
2837                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2838
2839         # Keeps track of counts across recursive calls
2840         if not tracker:
2841             tracker = dict(
2842                 running_total=0,
2843                 est_total=0,
2844                 current_page_thread=0,
2845                 total_parent_comments=0,
2846                 total_reply_comments=0)
2847
2848         # TODO: Deprecated
2849         # YouTube comments have a max depth of 2
2850         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2851         if max_depth:
2852             self._downloader.deprecation_warning(
2853                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2854         if max_depth == 1 and parent:
2855             return
2856
2857         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2858             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2859
2860         continuation = self._extract_continuation(root_continuation_data)
2861
2862         response = None
2863         is_forced_continuation = False
2864         is_first_continuation = parent is None
2865         if is_first_continuation and not continuation:
2866             # Sometimes you can get comments by generating the continuation yourself,
2867             # even if YouTube initially reports them being disabled - e.g. stories comments.
2868             # Note: if the comment section is actually disabled, YouTube may return a response with
2869             # required check_get_keys missing. So we will disable that check initially in this case.
2870             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2871             is_forced_continuation = True
2872
2873         for page_num in itertools.count(0):
2874             if not continuation:
2875                 break
2876             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2877             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2878             if page_num == 0:
2879                 if is_first_continuation:
2880                     note_prefix = 'Downloading comment section API JSON'
2881                 else:
2882                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2883                         tracker['current_page_thread'], comment_prog_str)
2884             else:
2885                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2886                     '       ' if parent else '', ' replies' if parent else '',
2887                     page_num, comment_prog_str)
2888
2889             response = self._extract_response(
2890                 item_id=None, query=continuation,
2891                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2892                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2893             is_forced_continuation = False
2894             continuation_contents = traverse_obj(
2895                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2896
2897             continuation = None
2898             for continuation_section in continuation_contents:
2899                 continuation_items = traverse_obj(
2900                     continuation_section,
2901                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2902                     get_all=False, expected_type=list) or []
2903                 if is_first_continuation:
2904                     continuation = extract_header(continuation_items)
2905                     is_first_continuation = False
2906                     if continuation:
2907                         break
2908                     continue
2909
2910                 for entry in extract_thread(continuation_items):
2911                     if not entry:
2912                         return
2913                     yield entry
2914                 continuation = self._extract_continuation({'contents': continuation_items})
2915                 if continuation:
2916                     break
2917
2918         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2919         if message and not parent and tracker['running_total'] == 0:
2920             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2921
2922     @staticmethod
2923     def _generate_comment_continuation(video_id):
2924         """
2925         Generates initial comment section continuation token from given video id
2926         """
2927         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2928         return base64.b64encode(token.encode()).decode()
2929
2930     def _get_comments(self, ytcfg, video_id, contents, webpage):
2931         """Entry for comment extraction"""
2932         def _real_comment_extract(contents):
2933             renderer = next((
2934                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2935                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2936             yield from self._comment_entries(renderer, ytcfg, video_id)
2937
2938         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2939         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2940
2941     @staticmethod
2942     def _get_checkok_params():
2943         return {'contentCheckOk': True, 'racyCheckOk': True}
2944
2945     @classmethod
2946     def _generate_player_context(cls, sts=None):
2947         context = {
2948             'html5Preference': 'HTML5_PREF_WANTS',
2949         }
2950         if sts is not None:
2951             context['signatureTimestamp'] = sts
2952         return {
2953             'playbackContext': {
2954                 'contentPlaybackContext': context
2955             },
2956             **cls._get_checkok_params()
2957         }
2958
2959     @staticmethod
2960     def _is_agegated(player_response):
2961         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2962             return True
2963
2964         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2965         AGE_GATE_REASONS = (
2966             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2967             'age_verification_required', 'age_check_required',  # status
2968         )
2969         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
2970
2971     @staticmethod
2972     def _is_unplayable(player_response):
2973         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
2974
2975     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
2976
2977         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
2978         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
2979         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
2980         headers = self.generate_api_headers(
2981             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
2982
2983         yt_query = {
2984             'videoId': video_id,
2985             'params': '8AEB'  # enable stories
2986         }
2987         yt_query.update(self._generate_player_context(sts))
2988         return self._extract_response(
2989             item_id=video_id, ep='player', query=yt_query,
2990             ytcfg=player_ytcfg, headers=headers, fatal=True,
2991             default_client=client,
2992             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
2993         ) or None
2994
2995     def _get_requested_clients(self, url, smuggled_data):
2996         requested_clients = []
2997         default = ['android', 'web']
2998         allowed_clients = sorted(
2999             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3000             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3001         for client in self._configuration_arg('player_client'):
3002             if client in allowed_clients:
3003                 requested_clients.append(client)
3004             elif client == 'default':
3005                 requested_clients.extend(default)
3006             elif client == 'all':
3007                 requested_clients.extend(allowed_clients)
3008             else:
3009                 self.report_warning(f'Skipping unsupported client {client}')
3010         if not requested_clients:
3011             requested_clients = default
3012
3013         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3014             requested_clients.extend(
3015                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3016
3017         return orderedSet(requested_clients)
3018
3019     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3020         initial_pr = None
3021         if webpage:
3022             initial_pr = self._extract_yt_initial_variable(
3023                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
3024                 video_id, 'initial player response')
3025
3026         all_clients = set(clients)
3027         clients = clients[::-1]
3028         prs = []
3029
3030         def append_client(*client_names):
3031             """ Append the first client name that exists but not already used """
3032             for client_name in client_names:
3033                 actual_client = _split_innertube_client(client_name)[0]
3034                 if actual_client in INNERTUBE_CLIENTS:
3035                     if actual_client not in all_clients:
3036                         clients.append(client_name)
3037                         all_clients.add(actual_client)
3038                         return
3039
3040         # Android player_response does not have microFormats which are needed for
3041         # extraction of some data. So we return the initial_pr with formats
3042         # stripped out even if not requested by the user
3043         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3044         if initial_pr:
3045             pr = dict(initial_pr)
3046             pr['streamingData'] = None
3047             prs.append(pr)
3048
3049         last_error = None
3050         tried_iframe_fallback = False
3051         player_url = None
3052         while clients:
3053             client, base_client, variant = _split_innertube_client(clients.pop())
3054             player_ytcfg = master_ytcfg if client == 'web' else {}
3055             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3056                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3057
3058             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3059             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3060             if 'js' in self._configuration_arg('player_skip'):
3061                 require_js_player = False
3062                 player_url = None
3063
3064             if not player_url and not tried_iframe_fallback and require_js_player:
3065                 player_url = self._download_player_url(video_id)
3066                 tried_iframe_fallback = True
3067
3068             try:
3069                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3070                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3071             except ExtractorError as e:
3072                 if last_error:
3073                     self.report_warning(last_error)
3074                 last_error = e
3075                 continue
3076
3077             if pr:
3078                 prs.append(pr)
3079
3080             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3081             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3082                 append_client(f'{base_client}_creator')
3083             elif self._is_agegated(pr):
3084                 if variant == 'tv_embedded':
3085                     append_client(f'{base_client}_embedded')
3086                 elif not variant:
3087                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3088
3089         if last_error:
3090             if not len(prs):
3091                 raise last_error
3092             self.report_warning(last_error)
3093         return prs, player_url
3094
3095     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3096         itags, stream_ids = {}, []
3097         itag_qualities, res_qualities = {}, {}
3098         q = qualities([
3099             # Normally tiny is the smallest video-only formats. But
3100             # audio-only formats with unknown quality may get tagged as tiny
3101             'tiny',
3102             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3103             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3104         ])
3105         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3106
3107         for fmt in streaming_formats:
3108             if fmt.get('targetDurationSec'):
3109                 continue
3110
3111             itag = str_or_none(fmt.get('itag'))
3112             audio_track = fmt.get('audioTrack') or {}
3113             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3114             if stream_id in stream_ids:
3115                 continue
3116
3117             quality = fmt.get('quality')
3118             height = int_or_none(fmt.get('height'))
3119             if quality == 'tiny' or not quality:
3120                 quality = fmt.get('audioQuality', '').lower() or quality
3121             # The 3gp format (17) in android client has a quality of "small",
3122             # but is actually worse than other formats
3123             if itag == '17':
3124                 quality = 'tiny'
3125             if quality:
3126                 if itag:
3127                     itag_qualities[itag] = quality
3128                 if height:
3129                     res_qualities[height] = quality
3130             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3131             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3132             # number of fragment that would subsequently requested with (`&sq=N`)
3133             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3134                 continue
3135
3136             fmt_url = fmt.get('url')
3137             if not fmt_url:
3138                 sc = compat_parse_qs(fmt.get('signatureCipher'))
3139                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3140                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3141                 if not (sc and fmt_url and encrypted_sig):
3142                     continue
3143                 if not player_url:
3144                     continue
3145                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
3146                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
3147                 fmt_url += '&' + sp + '=' + signature
3148
3149             query = parse_qs(fmt_url)
3150             throttled = False
3151             if query.get('n'):
3152                 try:
3153                     fmt_url = update_url_query(fmt_url, {
3154                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3155                 except ExtractorError as e:
3156                     self.report_warning(
3157                         'nsig extraction failed: You may experience throttling for some formats\n'
3158                         f'n = {query["n"][0]} ; player = {player_url}\n{e}', only_once=True)
3159                     throttled = True
3160
3161             if itag:
3162                 itags[itag] = 'https'
3163                 stream_ids.append(stream_id)
3164
3165             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3166             language_preference = (
3167                 10 if audio_track.get('audioIsDefault') and 10
3168                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3169                 else -1)
3170             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3171             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3172             # Make sure to avoid false positives with small duration differences.
3173             # Eg: __2ABJjxzNo, ySuUZEjARPY
3174             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3175             if is_damaged:
3176                 self.report_warning(
3177                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3178             dct = {
3179                 'asr': int_or_none(fmt.get('audioSampleRate')),
3180                 'filesize': int_or_none(fmt.get('contentLength')),
3181                 'format_id': itag,
3182                 'format_note': join_nonempty(
3183                     '%s%s' % (audio_track.get('displayName') or '',
3184                               ' (default)' if language_preference > 0 else ''),
3185                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3186                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3187                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3188                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3189                 'fps': int_or_none(fmt.get('fps')) or None,
3190                 'height': height,
3191                 'quality': q(quality),
3192                 'has_drm': bool(fmt.get('drmFamilies')),
3193                 'tbr': tbr,
3194                 'url': fmt_url,
3195                 'width': int_or_none(fmt.get('width')),
3196                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3197                                           'desc' if language_preference < -1 else ''),
3198                 'language_preference': language_preference,
3199                 # Strictly de-prioritize damaged and 3gp formats
3200                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3201             }
3202             mime_mobj = re.match(
3203                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3204             if mime_mobj:
3205                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3206                 dct.update(parse_codecs(mime_mobj.group(2)))
3207             no_audio = dct.get('acodec') == 'none'
3208             no_video = dct.get('vcodec') == 'none'
3209             if no_audio:
3210                 dct['vbr'] = tbr
3211             if no_video:
3212                 dct['abr'] = tbr
3213             if no_audio or no_video:
3214                 dct['downloader_options'] = {
3215                     # Youtube throttles chunks >~10M
3216                     'http_chunk_size': 10485760,
3217                 }
3218                 if dct.get('ext'):
3219                     dct['container'] = dct['ext'] + '_dash'
3220             yield dct
3221
3222         live_from_start = is_live and self.get_param('live_from_start')
3223         skip_manifests = self._configuration_arg('skip')
3224         if not self.get_param('youtube_include_hls_manifest', True):
3225             skip_manifests.append('hls')
3226         if not self.get_param('youtube_include_dash_manifest', True):
3227             skip_manifests.append('dash')
3228         get_dash = 'dash' not in skip_manifests and (
3229             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3230         get_hls = not live_from_start and 'hls' not in skip_manifests
3231
3232         def process_manifest_format(f, proto, itag):
3233             if itag in itags:
3234                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3235                     return False
3236                 itag = f'{itag}-{proto}'
3237             if itag:
3238                 f['format_id'] = itag
3239                 itags[itag] = proto
3240
3241             f['quality'] = next((
3242                 q(qdict[val])
3243                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3244                 if val in qdict), -1)
3245             return True
3246
3247         for sd in streaming_data:
3248             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3249             if hls_manifest_url:
3250                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3251                     if process_manifest_format(f, 'hls', self._search_regex(
3252                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3253                         yield f
3254
3255             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3256             if dash_manifest_url:
3257                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3258                     if process_manifest_format(f, 'dash', f['format_id']):
3259                         f['filesize'] = int_or_none(self._search_regex(
3260                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3261                         if live_from_start:
3262                             f['is_from_start'] = True
3263
3264                         yield f
3265
3266     def _extract_storyboard(self, player_responses, duration):
3267         spec = get_first(
3268             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3269         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3270         if not base_url:
3271             return
3272         L = len(spec) - 1
3273         for i, args in enumerate(spec):
3274             args = args.split('#')
3275             counts = list(map(int_or_none, args[:5]))
3276             if len(args) != 8 or not all(counts):
3277                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3278                 continue
3279             width, height, frame_count, cols, rows = counts
3280             N, sigh = args[6:]
3281
3282             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3283             fragment_count = frame_count / (cols * rows)
3284             fragment_duration = duration / fragment_count
3285             yield {
3286                 'format_id': f'sb{i}',
3287                 'format_note': 'storyboard',
3288                 'ext': 'mhtml',
3289                 'protocol': 'mhtml',
3290                 'acodec': 'none',
3291                 'vcodec': 'none',
3292                 'url': url,
3293                 'width': width,
3294                 'height': height,
3295                 'fragments': [{
3296                     'url': url.replace('$M', str(j)),
3297                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3298                 } for j in range(math.ceil(fragment_count))],
3299             }
3300
3301     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3302         webpage = None
3303         if 'webpage' not in self._configuration_arg('player_skip'):
3304             webpage = self._download_webpage(
3305                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3306
3307         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3308
3309         player_responses, player_url = self._extract_player_responses(
3310             self._get_requested_clients(url, smuggled_data),
3311             video_id, webpage, master_ytcfg)
3312
3313         return webpage, master_ytcfg, player_responses, player_url
3314
3315     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3316         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3317         is_live = get_first(video_details, 'isLive')
3318         if is_live is None:
3319             is_live = get_first(live_broadcast_details, 'isLiveNow')
3320
3321         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3322         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3323
3324         return live_broadcast_details, is_live, streaming_data, formats
3325
3326     def _real_extract(self, url):
3327         url, smuggled_data = unsmuggle_url(url, {})
3328         video_id = self._match_id(url)
3329
3330         base_url = self.http_scheme() + '//www.youtube.com/'
3331         webpage_url = base_url + 'watch?v=' + video_id
3332
3333         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3334
3335         playability_statuses = traverse_obj(
3336             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3337
3338         trailer_video_id = get_first(
3339             playability_statuses,
3340             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3341             expected_type=str)
3342         if trailer_video_id:
3343             return self.url_result(
3344                 trailer_video_id, self.ie_key(), trailer_video_id)
3345
3346         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3347                        if webpage else (lambda x: None))
3348
3349         video_details = traverse_obj(
3350             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3351         microformats = traverse_obj(
3352             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3353             expected_type=dict, default=[])
3354         video_title = (
3355             get_first(video_details, 'title')
3356             or self._get_text(microformats, (..., 'title'))
3357             or search_meta(['og:title', 'twitter:title', 'title']))
3358         video_description = get_first(video_details, 'shortDescription')
3359
3360         multifeed_metadata_list = get_first(
3361             player_responses,
3362             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3363             expected_type=str)
3364         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3365             if self.get_param('noplaylist'):
3366                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3367             else:
3368                 entries = []
3369                 feed_ids = []
3370                 for feed in multifeed_metadata_list.split(','):
3371                     # Unquote should take place before split on comma (,) since textual
3372                     # fields may contain comma as well (see
3373                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3374                     feed_data = compat_parse_qs(
3375                         compat_urllib_parse_unquote_plus(feed))
3376
3377                     def feed_entry(name):
3378                         return try_get(
3379                             feed_data, lambda x: x[name][0], compat_str)
3380
3381                     feed_id = feed_entry('id')
3382                     if not feed_id:
3383                         continue
3384                     feed_title = feed_entry('title')
3385                     title = video_title
3386                     if feed_title:
3387                         title += ' (%s)' % feed_title
3388                     entries.append({
3389                         '_type': 'url_transparent',
3390                         'ie_key': 'Youtube',
3391                         'url': smuggle_url(
3392                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3393                             {'force_singlefeed': True}),
3394                         'title': title,
3395                     })
3396                     feed_ids.append(feed_id)
3397                 self.to_screen(
3398                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3399                     % (', '.join(feed_ids), video_id))
3400                 return self.playlist_result(
3401                     entries, video_id, video_title, video_description)
3402
3403         duration = int_or_none(
3404             get_first(video_details, 'lengthSeconds')
3405             or get_first(microformats, 'lengthSeconds')
3406             or parse_duration(search_meta('duration'))) or None
3407
3408         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3409             video_id, microformats, video_details, player_responses, player_url, duration)
3410
3411         if not formats:
3412             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3413                 self.report_drm(video_id)
3414             pemr = get_first(
3415                 playability_statuses,
3416                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3417             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3418             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3419             if subreason:
3420                 if subreason == 'The uploader has not made this video available in your country.':
3421                     countries = get_first(microformats, 'availableCountries')
3422                     if not countries:
3423                         regions_allowed = search_meta('regionsAllowed')
3424                         countries = regions_allowed.split(',') if regions_allowed else None
3425                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3426                 reason += f'. {subreason}'
3427             if reason:
3428                 self.raise_no_formats(reason, expected=True)
3429
3430         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3431         if not keywords and webpage:
3432             keywords = [
3433                 unescapeHTML(m.group('content'))
3434                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3435         for keyword in keywords:
3436             if keyword.startswith('yt:stretch='):
3437                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3438                 if mobj:
3439                     # NB: float is intentional for forcing float division
3440                     w, h = (float(v) for v in mobj.groups())
3441                     if w > 0 and h > 0:
3442                         ratio = w / h
3443                         for f in formats:
3444                             if f.get('vcodec') != 'none':
3445                                 f['stretched_ratio'] = ratio
3446                         break
3447         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3448         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3449         if thumbnail_url:
3450             thumbnails.append({
3451                 'url': thumbnail_url,
3452             })
3453         original_thumbnails = thumbnails.copy()
3454
3455         # The best resolution thumbnails sometimes does not appear in the webpage
3456         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3457         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3458         thumbnail_names = [
3459             # While the *1,*2,*3 thumbnails are just below their correspnding "*default" variants
3460             # in resolution, these are not the custom thumbnail. So de-prioritize them
3461             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3462             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3463         ]
3464         n_thumbnail_names = len(thumbnail_names)
3465         thumbnails.extend({
3466             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3467                 video_id=video_id, name=name, ext=ext,
3468                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3469         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3470         for thumb in thumbnails:
3471             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3472             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3473         self._remove_duplicate_formats(thumbnails)
3474         self._downloader._sort_thumbnails(original_thumbnails)
3475
3476         category = get_first(microformats, 'category') or search_meta('genre')
3477         channel_id = str_or_none(
3478             get_first(video_details, 'channelId')
3479             or get_first(microformats, 'externalChannelId')
3480             or search_meta('channelId'))
3481         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3482
3483         live_content = get_first(video_details, 'isLiveContent')
3484         is_upcoming = get_first(video_details, 'isUpcoming')
3485         if is_live is None:
3486             if is_upcoming or live_content is False:
3487                 is_live = False
3488         if is_upcoming is None and (live_content or is_live):
3489             is_upcoming = False
3490         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3491         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3492         if not duration and live_end_time and live_start_time:
3493             duration = live_end_time - live_start_time
3494
3495         if is_live and self.get_param('live_from_start'):
3496             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3497
3498         formats.extend(self._extract_storyboard(player_responses, duration))
3499
3500         # Source is given priority since formats that throttle are given lower source_preference
3501         # When throttling issue is fully fixed, remove this
3502         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3503
3504         info = {
3505             'id': video_id,
3506             'title': video_title,
3507             'formats': formats,
3508             'thumbnails': thumbnails,
3509             # The best thumbnail that we are sure exists. Prevents unnecessary
3510             # URL checking if user don't care about getting the best possible thumbnail
3511             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3512             'description': video_description,
3513             'uploader': get_first(video_details, 'author'),
3514             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3515             'uploader_url': owner_profile_url,
3516             'channel_id': channel_id,
3517             'channel_url': format_field(channel_id, template='https://www.youtube.com/channel/%s'),
3518             'duration': duration,
3519             'view_count': int_or_none(
3520                 get_first((video_details, microformats), (..., 'viewCount'))
3521                 or search_meta('interactionCount')),
3522             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3523             'age_limit': 18 if (
3524                 get_first(microformats, 'isFamilySafe') is False
3525                 or search_meta('isFamilyFriendly') == 'false'
3526                 or search_meta('og:restrictions:age') == '18+') else 0,
3527             'webpage_url': webpage_url,
3528             'categories': [category] if category else None,
3529             'tags': keywords,
3530             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3531             'is_live': is_live,
3532             'was_live': (False if is_live or is_upcoming or live_content is False
3533                          else None if is_live is None or is_upcoming is None
3534                          else live_content),
3535             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3536             'release_timestamp': live_start_time,
3537         }
3538
3539         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3540         if pctr:
3541             def get_lang_code(track):
3542                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3543                         or track.get('languageCode'))
3544
3545             # Converted into dicts to remove duplicates
3546             captions = {
3547                 get_lang_code(sub): sub
3548                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3549             translation_languages = {
3550                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3551                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3552
3553             def process_language(container, base_url, lang_code, sub_name, query):
3554                 lang_subs = container.setdefault(lang_code, [])
3555                 for fmt in self._SUBTITLE_FORMATS:
3556                     query.update({
3557                         'fmt': fmt,
3558                     })
3559                     lang_subs.append({
3560                         'ext': fmt,
3561                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3562                         'name': sub_name,
3563                     })
3564
3565             subtitles, automatic_captions = {}, {}
3566             for lang_code, caption_track in captions.items():
3567                 base_url = caption_track.get('baseUrl')
3568                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3569                 if not base_url:
3570                     continue
3571                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3572                 if caption_track.get('kind') != 'asr':
3573                     if not lang_code:
3574                         continue
3575                     process_language(
3576                         subtitles, base_url, lang_code, lang_name, {})
3577                     if not caption_track.get('isTranslatable'):
3578                         continue
3579                 for trans_code, trans_name in translation_languages.items():
3580                     if not trans_code:
3581                         continue
3582                     orig_trans_code = trans_code
3583                     if caption_track.get('kind') != 'asr':
3584                         if 'translated_subs' in self._configuration_arg('skip'):
3585                             continue
3586                         trans_code += f'-{lang_code}'
3587                         trans_name += format_field(lang_name, template=' from %s')
3588                     # Add an "-orig" label to the original language so that it can be distinguished.
3589                     # The subs are returned without "-orig" as well for compatibility
3590                     if lang_code == f'a-{orig_trans_code}':
3591                         process_language(
3592                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3593                     # Setting tlang=lang returns damaged subtitles.
3594                     process_language(automatic_captions, base_url, trans_code, trans_name,
3595                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3596             info['automatic_captions'] = automatic_captions
3597             info['subtitles'] = subtitles
3598
3599         parsed_url = compat_urllib_parse_urlparse(url)
3600         for component in [parsed_url.fragment, parsed_url.query]:
3601             query = compat_parse_qs(component)
3602             for k, v in query.items():
3603                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3604                     d_k += '_time'
3605                     if d_k not in info and k in s_ks:
3606                         info[d_k] = parse_duration(query[k][0])
3607
3608         # Youtube Music Auto-generated description
3609         if video_description:
3610             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
3611             if mobj:
3612                 release_year = mobj.group('release_year')
3613                 release_date = mobj.group('release_date')
3614                 if release_date:
3615                     release_date = release_date.replace('-', '')
3616                     if not release_year:
3617                         release_year = release_date[:4]
3618                 info.update({
3619                     'album': mobj.group('album'.strip()),
3620                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3621                     'track': mobj.group('track').strip(),
3622                     'release_date': release_date,
3623                     'release_year': int_or_none(release_year),
3624                 })
3625
3626         initial_data = None
3627         if webpage:
3628             initial_data = self._extract_yt_initial_variable(
3629                 webpage, self._YT_INITIAL_DATA_RE, video_id,
3630                 'yt initial data')
3631         if not initial_data:
3632             query = {'videoId': video_id}
3633             query.update(self._get_checkok_params())
3634             initial_data = self._extract_response(
3635                 item_id=video_id, ep='next', fatal=False,
3636                 ytcfg=master_ytcfg, query=query,
3637                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3638                 note='Downloading initial data API JSON')
3639
3640         try:  # This will error if there is no livechat
3641             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3642         except (KeyError, IndexError, TypeError):
3643             pass
3644         else:
3645             info.setdefault('subtitles', {})['live_chat'] = [{
3646                 'url': f'https://www.youtube.com/watch?v={video_id}',  # url is needed to set cookies
3647                 'video_id': video_id,
3648                 'ext': 'json',
3649                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3650             }]
3651
3652         if initial_data:
3653             info['chapters'] = (
3654                 self._extract_chapters_from_json(initial_data, duration)
3655                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3656                 or None)
3657
3658         contents = traverse_obj(
3659             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3660             expected_type=list, default=[])
3661
3662         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3663         if vpir:
3664             stl = vpir.get('superTitleLink')
3665             if stl:
3666                 stl = self._get_text(stl)
3667                 if try_get(
3668                         vpir,
3669                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3670                     info['location'] = stl
3671                 else:
3672                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3673                     if mobj:
3674                         info.update({
3675                             'series': mobj.group(1),
3676                             'season_number': int(mobj.group(2)),
3677                             'episode_number': int(mobj.group(3)),
3678                         })
3679             for tlb in (try_get(
3680                     vpir,
3681                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3682                     list) or []):
3683                 tbr = tlb.get('toggleButtonRenderer') or {}
3684                 for getter, regex in [(
3685                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3686                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3687                             lambda x: x['accessibility'],
3688                             lambda x: x['accessibilityData']['accessibilityData'],
3689                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3690                     label = (try_get(tbr, getter, dict) or {}).get('label')
3691                     if label:
3692                         mobj = re.match(regex, label)
3693                         if mobj:
3694                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3695                             break
3696             sbr_tooltip = try_get(
3697                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3698             if sbr_tooltip:
3699                 like_count, dislike_count = sbr_tooltip.split(' / ')
3700                 info.update({
3701                     'like_count': str_to_int(like_count),
3702                     'dislike_count': str_to_int(dislike_count),
3703                 })
3704         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3705         if vsir:
3706             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3707             info.update({
3708                 'channel': self._get_text(vor, 'title'),
3709                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3710
3711             rows = try_get(
3712                 vsir,
3713                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3714                 list) or []
3715             multiple_songs = False
3716             for row in rows:
3717                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3718                     multiple_songs = True
3719                     break
3720             for row in rows:
3721                 mrr = row.get('metadataRowRenderer') or {}
3722                 mrr_title = mrr.get('title')
3723                 if not mrr_title:
3724                     continue
3725                 mrr_title = self._get_text(mrr, 'title')
3726                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3727                 if mrr_title == 'License':
3728                     info['license'] = mrr_contents_text
3729                 elif not multiple_songs:
3730                     if mrr_title == 'Album':
3731                         info['album'] = mrr_contents_text
3732                     elif mrr_title == 'Artist':
3733                         info['artist'] = mrr_contents_text
3734                     elif mrr_title == 'Song':
3735                         info['track'] = mrr_contents_text
3736
3737         fallbacks = {
3738             'channel': 'uploader',
3739             'channel_id': 'uploader_id',
3740             'channel_url': 'uploader_url',
3741         }
3742
3743         # The upload date for scheduled, live and past live streams / premieres in microformats
3744         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3745         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3746         upload_date = (
3747             unified_strdate(get_first(microformats, 'uploadDate'))
3748             or unified_strdate(search_meta('uploadDate')))
3749         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3750             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3751         info['upload_date'] = upload_date
3752
3753         for to, frm in fallbacks.items():
3754             if not info.get(to):
3755                 info[to] = info.get(frm)
3756
3757         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3758             v = info.get(s_k)
3759             if v:
3760                 info[d_k] = v
3761
3762         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3763         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3764         is_membersonly = None
3765         is_premium = None
3766         if initial_data and is_private is not None:
3767             is_membersonly = False
3768             is_premium = False
3769             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3770             badge_labels = set()
3771             for content in contents:
3772                 if not isinstance(content, dict):
3773                     continue
3774                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3775             for badge_label in badge_labels:
3776                 if badge_label.lower() == 'members only':
3777                     is_membersonly = True
3778                 elif badge_label.lower() == 'premium':
3779                     is_premium = True
3780                 elif badge_label.lower() == 'unlisted':
3781                     is_unlisted = True
3782
3783         info['availability'] = self._availability(
3784             is_private=is_private,
3785             needs_premium=is_premium,
3786             needs_subscription=is_membersonly,
3787             needs_auth=info['age_limit'] >= 18,
3788             is_unlisted=None if is_private is None else is_unlisted)
3789
3790         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3791
3792         self.mark_watched(video_id, player_responses)
3793
3794         return info
3795
3796
3797 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3798
3799     @staticmethod
3800     def passthrough_smuggled_data(func):
3801         def _smuggle(entries, smuggled_data):
3802             for entry in entries:
3803                 # TODO: Convert URL to music.youtube instead.
3804                 # Do we need to passthrough any other smuggled_data?
3805                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3806                 yield entry
3807
3808         @functools.wraps(func)
3809         def wrapper(self, url):
3810             url, smuggled_data = unsmuggle_url(url, {})
3811             if self.is_music_url(url):
3812                 smuggled_data['is_music_url'] = True
3813             info_dict = func(self, url, smuggled_data)
3814             if smuggled_data and info_dict.get('entries'):
3815                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3816             return info_dict
3817         return wrapper
3818
3819     def _extract_channel_id(self, webpage):
3820         channel_id = self._html_search_meta(
3821             'channelId', webpage, 'channel id', default=None)
3822         if channel_id:
3823             return channel_id
3824         channel_url = self._html_search_meta(
3825             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3826              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3827              'twitter:app:url:googleplay'), webpage, 'channel url')
3828         return self._search_regex(
3829             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3830             channel_url, 'channel id')
3831
3832     @staticmethod
3833     def _extract_basic_item_renderer(item):
3834         # Modified from _extract_grid_item_renderer
3835         known_basic_renderers = (
3836             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3837         )
3838         for key, renderer in item.items():
3839             if not isinstance(renderer, dict):
3840                 continue
3841             elif key in known_basic_renderers:
3842                 return renderer
3843             elif key.startswith('grid') and key.endswith('Renderer'):
3844                 return renderer
3845
3846     def _grid_entries(self, grid_renderer):
3847         for item in grid_renderer['items']:
3848             if not isinstance(item, dict):
3849                 continue
3850             renderer = self._extract_basic_item_renderer(item)
3851             if not isinstance(renderer, dict):
3852                 continue
3853             title = self._get_text(renderer, 'title')
3854
3855             # playlist
3856             playlist_id = renderer.get('playlistId')
3857             if playlist_id:
3858                 yield self.url_result(
3859                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3860                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3861                     video_title=title)
3862                 continue
3863             # video
3864             video_id = renderer.get('videoId')
3865             if video_id:
3866                 yield self._extract_video(renderer)
3867                 continue
3868             # channel
3869             channel_id = renderer.get('channelId')
3870             if channel_id:
3871                 yield self.url_result(
3872                     'https://www.youtube.com/channel/%s' % channel_id,
3873                     ie=YoutubeTabIE.ie_key(), video_title=title)
3874                 continue
3875             # generic endpoint URL support
3876             ep_url = urljoin('https://www.youtube.com/', try_get(
3877                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3878                 compat_str))
3879             if ep_url:
3880                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3881                     if ie.suitable(ep_url):
3882                         yield self.url_result(
3883                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3884                         break
3885
3886     def _music_reponsive_list_entry(self, renderer):
3887         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3888         if video_id:
3889             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3890                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3891         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3892         if playlist_id:
3893             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3894             if video_id:
3895                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3896                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3897             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3898                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3899         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3900         if browse_id:
3901             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3902                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3903
3904     def _shelf_entries_from_content(self, shelf_renderer):
3905         content = shelf_renderer.get('content')
3906         if not isinstance(content, dict):
3907             return
3908         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3909         if renderer:
3910             # TODO: add support for nested playlists so each shelf is processed
3911             # as separate playlist
3912             # TODO: this includes only first N items
3913             yield from self._grid_entries(renderer)
3914         renderer = content.get('horizontalListRenderer')
3915         if renderer:
3916             # TODO
3917             pass
3918
3919     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3920         ep = try_get(
3921             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3922             compat_str)
3923         shelf_url = urljoin('https://www.youtube.com', ep)
3924         if shelf_url:
3925             # Skipping links to another channels, note that checking for
3926             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3927             # will not work
3928             if skip_channels and '/channels?' in shelf_url:
3929                 return
3930             title = self._get_text(shelf_renderer, 'title')
3931             yield self.url_result(shelf_url, video_title=title)
3932         # Shelf may not contain shelf URL, fallback to extraction from content
3933         yield from self._shelf_entries_from_content(shelf_renderer)
3934
3935     def _playlist_entries(self, video_list_renderer):
3936         for content in video_list_renderer['contents']:
3937             if not isinstance(content, dict):
3938                 continue
3939             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3940             if not isinstance(renderer, dict):
3941                 continue
3942             video_id = renderer.get('videoId')
3943             if not video_id:
3944                 continue
3945             yield self._extract_video(renderer)
3946
3947     def _rich_entries(self, rich_grid_renderer):
3948         renderer = try_get(
3949             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3950         video_id = renderer.get('videoId')
3951         if not video_id:
3952             return
3953         yield self._extract_video(renderer)
3954
3955     def _video_entry(self, video_renderer):
3956         video_id = video_renderer.get('videoId')
3957         if video_id:
3958             return self._extract_video(video_renderer)
3959
3960     def _hashtag_tile_entry(self, hashtag_tile_renderer):
3961         url = urljoin('https://youtube.com', traverse_obj(
3962             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
3963         if url:
3964             return self.url_result(
3965                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
3966
3967     def _post_thread_entries(self, post_thread_renderer):
3968         post_renderer = try_get(
3969             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
3970         if not post_renderer:
3971             return
3972         # video attachment
3973         video_renderer = try_get(
3974             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
3975         video_id = video_renderer.get('videoId')
3976         if video_id:
3977             entry = self._extract_video(video_renderer)
3978             if entry:
3979                 yield entry
3980         # playlist attachment
3981         playlist_id = try_get(
3982             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
3983         if playlist_id:
3984             yield self.url_result(
3985                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
3986                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3987         # inline video links
3988         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
3989         for run in runs:
3990             if not isinstance(run, dict):
3991                 continue
3992             ep_url = try_get(
3993                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
3994             if not ep_url:
3995                 continue
3996             if not YoutubeIE.suitable(ep_url):
3997                 continue
3998             ep_video_id = YoutubeIE._match_id(ep_url)
3999             if video_id == ep_video_id:
4000                 continue
4001             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4002
4003     def _post_thread_continuation_entries(self, post_thread_continuation):
4004         contents = post_thread_continuation.get('contents')
4005         if not isinstance(contents, list):
4006             return
4007         for content in contents:
4008             renderer = content.get('backstagePostThreadRenderer')
4009             if not isinstance(renderer, dict):
4010                 continue
4011             yield from self._post_thread_entries(renderer)
4012
4013     r''' # unused
4014     def _rich_grid_entries(self, contents):
4015         for content in contents:
4016             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4017             if video_renderer:
4018                 entry = self._video_entry(video_renderer)
4019                 if entry:
4020                     yield entry
4021     '''
4022
4023     def _extract_entries(self, parent_renderer, continuation_list):
4024         # continuation_list is modified in-place with continuation_list = [continuation_token]
4025         continuation_list[:] = [None]
4026         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4027         for content in contents:
4028             if not isinstance(content, dict):
4029                 continue
4030             is_renderer = traverse_obj(
4031                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4032                 expected_type=dict)
4033             if not is_renderer:
4034                 renderer = content.get('richItemRenderer')
4035                 if renderer:
4036                     for entry in self._rich_entries(renderer):
4037                         yield entry
4038                     continuation_list[0] = self._extract_continuation(parent_renderer)
4039                 continue
4040             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4041             for isr_content in isr_contents:
4042                 if not isinstance(isr_content, dict):
4043                     continue
4044
4045                 known_renderers = {
4046                     'playlistVideoListRenderer': self._playlist_entries,
4047                     'gridRenderer': self._grid_entries,
4048                     'reelShelfRenderer': self._grid_entries,
4049                     'shelfRenderer': self._shelf_entries,
4050                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4051                     'backstagePostThreadRenderer': self._post_thread_entries,
4052                     'videoRenderer': lambda x: [self._video_entry(x)],
4053                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4054                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4055                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4056                 }
4057                 for key, renderer in isr_content.items():
4058                     if key not in known_renderers:
4059                         continue
4060                     for entry in known_renderers[key](renderer):
4061                         if entry:
4062                             yield entry
4063                     continuation_list[0] = self._extract_continuation(renderer)
4064                     break
4065
4066             if not continuation_list[0]:
4067                 continuation_list[0] = self._extract_continuation(is_renderer)
4068
4069         if not continuation_list[0]:
4070             continuation_list[0] = self._extract_continuation(parent_renderer)
4071
4072     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4073         continuation_list = [None]
4074         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4075         tab_content = try_get(tab, lambda x: x['content'], dict)
4076         if not tab_content:
4077             return
4078         parent_renderer = (
4079             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4080             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4081         yield from extract_entries(parent_renderer)
4082         continuation = continuation_list[0]
4083
4084         for page_num in itertools.count(1):
4085             if not continuation:
4086                 break
4087             headers = self.generate_api_headers(
4088                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4089             response = self._extract_response(
4090                 item_id=f'{item_id} page {page_num}',
4091                 query=continuation, headers=headers, ytcfg=ytcfg,
4092                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4093
4094             if not response:
4095                 break
4096             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4097             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4098             visitor_data = self._extract_visitor_data(response) or visitor_data
4099
4100             known_continuation_renderers = {
4101                 'playlistVideoListContinuation': self._playlist_entries,
4102                 'gridContinuation': self._grid_entries,
4103                 'itemSectionContinuation': self._post_thread_continuation_entries,
4104                 'sectionListContinuation': extract_entries,  # for feeds
4105             }
4106             continuation_contents = try_get(
4107                 response, lambda x: x['continuationContents'], dict) or {}
4108             continuation_renderer = None
4109             for key, value in continuation_contents.items():
4110                 if key not in known_continuation_renderers:
4111                     continue
4112                 continuation_renderer = value
4113                 continuation_list = [None]
4114                 yield from known_continuation_renderers[key](continuation_renderer)
4115                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4116                 break
4117             if continuation_renderer:
4118                 continue
4119
4120             known_renderers = {
4121                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4122                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4123                 'gridVideoRenderer': (self._grid_entries, 'items'),
4124                 'gridChannelRenderer': (self._grid_entries, 'items'),
4125                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4126                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4127                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4128                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4129             }
4130             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4131             continuation_items = try_get(
4132                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4133             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4134             video_items_renderer = None
4135             for key, value in continuation_item.items():
4136                 if key not in known_renderers:
4137                     continue
4138                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4139                 continuation_list = [None]
4140                 yield from known_renderers[key][0](video_items_renderer)
4141                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4142                 break
4143             if video_items_renderer:
4144                 continue
4145             break
4146
4147     @staticmethod
4148     def _extract_selected_tab(tabs, fatal=True):
4149         for tab in tabs:
4150             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4151             if renderer.get('selected') is True:
4152                 return renderer
4153         else:
4154             if fatal:
4155                 raise ExtractorError('Unable to find selected tab')
4156
4157     def _extract_uploader(self, data):
4158         uploader = {}
4159         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4160         owner = try_get(
4161             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4162         if owner:
4163             owner_text = owner.get('text')
4164             uploader['uploader'] = self._search_regex(
4165                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4166             uploader['uploader_id'] = try_get(
4167                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
4168             uploader['uploader_url'] = urljoin(
4169                 'https://www.youtube.com/',
4170                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
4171         return {k: v for k, v in uploader.items() if v is not None}
4172
4173     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4174         playlist_id = title = description = channel_url = channel_name = channel_id = None
4175         tags = []
4176
4177         selected_tab = self._extract_selected_tab(tabs)
4178         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4179         renderer = try_get(
4180             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4181         if renderer:
4182             channel_name = renderer.get('title')
4183             channel_url = renderer.get('channelUrl')
4184             channel_id = renderer.get('externalId')
4185         else:
4186             renderer = try_get(
4187                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4188
4189         if renderer:
4190             title = renderer.get('title')
4191             description = renderer.get('description', '')
4192             playlist_id = channel_id
4193             tags = renderer.get('keywords', '').split()
4194
4195         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4196         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4197         def _get_uncropped(url):
4198             return url_or_none((url or '').split('=')[0] + '=s0')
4199
4200         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4201         if avatar_thumbnails:
4202             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4203             if uncropped_avatar:
4204                 avatar_thumbnails.append({
4205                     'url': uncropped_avatar,
4206                     'id': 'avatar_uncropped',
4207                     'preference': 1
4208                 })
4209
4210         channel_banners = self._extract_thumbnails(
4211             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4212         for banner in channel_banners:
4213             banner['preference'] = -10
4214
4215         if channel_banners:
4216             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4217             if uncropped_banner:
4218                 channel_banners.append({
4219                     'url': uncropped_banner,
4220                     'id': 'banner_uncropped',
4221                     'preference': -5
4222                 })
4223
4224         primary_thumbnails = self._extract_thumbnails(
4225             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4226
4227         if playlist_id is None:
4228             playlist_id = item_id
4229
4230         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4231         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4232         if title is None:
4233             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4234         title += format_field(selected_tab, 'title', ' - %s')
4235         title += format_field(selected_tab, 'expandedText', ' - %s')
4236
4237         metadata = {
4238             'playlist_id': playlist_id,
4239             'playlist_title': title,
4240             'playlist_description': description,
4241             'uploader': channel_name,
4242             'uploader_id': channel_id,
4243             'uploader_url': channel_url,
4244             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4245             'tags': tags,
4246             'view_count': self._get_count(playlist_stats, 1),
4247             'availability': self._extract_availability(data),
4248             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4249             'playlist_count': self._get_count(playlist_stats, 0),
4250             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4251         }
4252         if not channel_id:
4253             metadata.update(self._extract_uploader(data))
4254         metadata.update({
4255             'channel': metadata['uploader'],
4256             'channel_id': metadata['uploader_id'],
4257             'channel_url': metadata['uploader_url']})
4258         return self.playlist_result(
4259             self._entries(
4260                 selected_tab, playlist_id, ytcfg,
4261                 self._extract_account_syncid(ytcfg, data),
4262                 self._extract_visitor_data(data, ytcfg)),
4263             **metadata)
4264
4265     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4266         first_id = last_id = response = None
4267         for page_num in itertools.count(1):
4268             videos = list(self._playlist_entries(playlist))
4269             if not videos:
4270                 return
4271             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4272             if start >= len(videos):
4273                 return
4274             for video in videos[start:]:
4275                 yield video
4276             first_id = first_id or videos[0]['id']
4277             last_id = videos[-1]['id']
4278             watch_endpoint = try_get(
4279                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4280             headers = self.generate_api_headers(
4281                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4282                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4283             query = {
4284                 'playlistId': playlist_id,
4285                 'videoId': watch_endpoint.get('videoId') or last_id,
4286                 'index': watch_endpoint.get('index') or len(videos),
4287                 'params': watch_endpoint.get('params') or 'OAE%3D'
4288             }
4289             response = self._extract_response(
4290                 item_id='%s page %d' % (playlist_id, page_num),
4291                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4292                 check_get_keys='contents'
4293             )
4294             playlist = try_get(
4295                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4296
4297     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4298         title = playlist.get('title') or try_get(
4299             data, lambda x: x['titleText']['simpleText'], compat_str)
4300         playlist_id = playlist.get('playlistId') or item_id
4301
4302         # Delegating everything except mix playlists to regular tab-based playlist URL
4303         playlist_url = urljoin(url, try_get(
4304             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4305             compat_str))
4306
4307         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4308         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4309         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4310
4311         if playlist_url and playlist_url != url and not is_known_unviewable:
4312             return self.url_result(
4313                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4314                 video_title=title)
4315
4316         return self.playlist_result(
4317             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4318             playlist_id=playlist_id, playlist_title=title)
4319
4320     def _extract_availability(self, data):
4321         """
4322         Gets the availability of a given playlist/tab.
4323         Note: Unless YouTube tells us explicitly, we do not assume it is public
4324         @param data: response
4325         """
4326         is_private = is_unlisted = None
4327         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4328         badge_labels = self._extract_badges(renderer)
4329
4330         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4331         privacy_dropdown_entries = try_get(
4332             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4333         for renderer_dict in privacy_dropdown_entries:
4334             is_selected = try_get(
4335                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4336             if not is_selected:
4337                 continue
4338             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4339             if label:
4340                 badge_labels.add(label.lower())
4341                 break
4342
4343         for badge_label in badge_labels:
4344             if badge_label == 'unlisted':
4345                 is_unlisted = True
4346             elif badge_label == 'private':
4347                 is_private = True
4348             elif badge_label == 'public':
4349                 is_unlisted = is_private = False
4350         return self._availability(is_private, False, False, False, is_unlisted)
4351
4352     @staticmethod
4353     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4354         sidebar_renderer = try_get(
4355             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4356         for item in sidebar_renderer:
4357             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4358             if renderer:
4359                 return renderer
4360
4361     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4362         """
4363         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4364         """
4365         browse_id = params = None
4366         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4367         if not renderer:
4368             return
4369         menu_renderer = try_get(
4370             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4371         for menu_item in menu_renderer:
4372             if not isinstance(menu_item, dict):
4373                 continue
4374             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4375             text = try_get(
4376                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4377             if not text or text.lower() != 'show unavailable videos':
4378                 continue
4379             browse_endpoint = try_get(
4380                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4381             browse_id = browse_endpoint.get('browseId')
4382             params = browse_endpoint.get('params')
4383             break
4384
4385         headers = self.generate_api_headers(
4386             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4387             visitor_data=self._extract_visitor_data(data, ytcfg))
4388         query = {
4389             'params': params or 'wgYCCAA=',
4390             'browseId': browse_id or 'VL%s' % item_id
4391         }
4392         return self._extract_response(
4393             item_id=item_id, headers=headers, query=query,
4394             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4395             note='Downloading API JSON with unavailable videos')
4396
4397     @property
4398     def skip_webpage(self):
4399         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4400
4401     def _extract_webpage(self, url, item_id, fatal=True):
4402         retries = self.get_param('extractor_retries', 3)
4403         count = -1
4404         webpage = data = last_error = None
4405         while count < retries:
4406             count += 1
4407             # Sometimes youtube returns a webpage with incomplete ytInitialData
4408             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4409             if last_error:
4410                 self.report_warning('%s. Retrying ...' % last_error)
4411             try:
4412                 webpage = self._download_webpage(
4413                     url, item_id,
4414                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4415                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4416             except ExtractorError as e:
4417                 if isinstance(e.cause, network_exceptions):
4418                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4419                         last_error = error_to_compat_str(e.cause or e.msg)
4420                         if count < retries:
4421                             continue
4422                 if fatal:
4423                     raise
4424                 self.report_warning(error_to_compat_str(e))
4425                 break
4426             else:
4427                 try:
4428                     self._extract_and_report_alerts(data)
4429                 except ExtractorError as e:
4430                     if fatal:
4431                         raise
4432                     self.report_warning(error_to_compat_str(e))
4433                     break
4434
4435                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4436                     break
4437
4438                 last_error = 'Incomplete yt initial data received'
4439                 if count >= retries:
4440                     if fatal:
4441                         raise ExtractorError(last_error)
4442                     self.report_warning(last_error)
4443                     break
4444
4445         return webpage, data
4446
4447     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4448         """Use if failed to extract ytcfg (and data) from initial webpage"""
4449         if not ytcfg and self.is_authenticated:
4450             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4451             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4452                 raise ExtractorError(
4453                     f'{msg}. If you are not downloading private content, or '
4454                     'your cookies are only for the first account and channel,'
4455                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4456                     expected=True)
4457             self.report_warning(msg, only_once=True)
4458
4459     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4460         data = None
4461         if not self.skip_webpage:
4462             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4463             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4464             # Reject webpage data if redirected to home page without explicitly requesting
4465             selected_tab = self._extract_selected_tab(traverse_obj(
4466                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4467             if (url != 'https://www.youtube.com/feed/recommended'
4468                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4469                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4470                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4471                 if fatal:
4472                     raise ExtractorError(msg, expected=True)
4473                 self.report_warning(msg, only_once=True)
4474         if not data:
4475             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4476             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4477         return data, ytcfg
4478
4479     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4480         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4481         resolve_response = self._extract_response(
4482             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4483             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4484         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4485         for ep_key, ep in endpoints.items():
4486             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4487             if params:
4488                 return self._extract_response(
4489                     item_id=item_id, query=params, ep=ep, headers=headers,
4490                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4491                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4492         err_note = 'Failed to resolve url (does the playlist exist?)'
4493         if fatal:
4494             raise ExtractorError(err_note, expected=True)
4495         self.report_warning(err_note, item_id)
4496
4497     _SEARCH_PARAMS = None
4498
4499     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4500         data = {'query': query}
4501         if params is NO_DEFAULT:
4502             params = self._SEARCH_PARAMS
4503         if params:
4504             data['params'] = params
4505
4506         content_keys = (
4507             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4508             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4509             # ytmusic search
4510             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4511             ('continuationContents', ),
4512         )
4513         display_id = f'query "{query}"'
4514         check_get_keys = tuple({keys[0] for keys in content_keys})
4515         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4516         self._report_playlist_authcheck(ytcfg, fatal=False)
4517
4518         continuation_list = [None]
4519         search = None
4520         for page_num in itertools.count(1):
4521             data.update(continuation_list[0] or {})
4522             headers = self.generate_api_headers(
4523                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4524             search = self._extract_response(
4525                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4526                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4527             slr_contents = traverse_obj(search, *content_keys)
4528             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4529             if not continuation_list[0]:
4530                 break
4531
4532
4533 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4534     IE_DESC = 'YouTube Tabs'
4535     _VALID_URL = r'''(?x:
4536         https?://
4537             (?:\w+\.)?
4538             (?:
4539                 youtube(?:kids)?\.com|
4540                 %(invidious)s
4541             )/
4542             (?:
4543                 (?P<channel_type>channel|c|user|browse)/|
4544                 (?P<not_channel>
4545                     feed/|hashtag/|
4546                     (?:playlist|watch)\?.*?\blist=
4547                 )|
4548                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4549             )
4550             (?P<id>[^/?\#&]+)
4551     )''' % {
4552         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4553         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4554     }
4555     IE_NAME = 'youtube:tab'
4556
4557     _TESTS = [{
4558         'note': 'playlists, multipage',
4559         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4560         'playlist_mincount': 94,
4561         'info_dict': {
4562             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4563             'title': 'Igor Kleiner - Playlists',
4564             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4565             'uploader': 'Igor Kleiner',
4566             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4567             'channel': 'Igor Kleiner',
4568             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4569             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4570             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4571             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4572             'channel_follower_count': int
4573         },
4574     }, {
4575         'note': 'playlists, multipage, different order',
4576         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4577         'playlist_mincount': 94,
4578         'info_dict': {
4579             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4580             'title': 'Igor Kleiner - Playlists',
4581             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4582             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4583             'uploader': 'Igor Kleiner',
4584             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4585             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4586             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4587             'channel': 'Igor Kleiner',
4588             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4589             'channel_follower_count': int
4590         },
4591     }, {
4592         'note': 'playlists, series',
4593         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4594         'playlist_mincount': 5,
4595         'info_dict': {
4596             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4597             'title': '3Blue1Brown - Playlists',
4598             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4599             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4600             'uploader': '3Blue1Brown',
4601             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4602             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4603             'channel': '3Blue1Brown',
4604             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4605             'tags': ['Mathematics'],
4606             'channel_follower_count': int
4607         },
4608     }, {
4609         'note': 'playlists, singlepage',
4610         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4611         'playlist_mincount': 4,
4612         'info_dict': {
4613             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4614             'title': 'ThirstForScience - Playlists',
4615             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4616             'uploader': 'ThirstForScience',
4617             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4618             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4619             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4620             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4621             'tags': 'count:13',
4622             'channel': 'ThirstForScience',
4623             'channel_follower_count': int
4624         }
4625     }, {
4626         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4627         'only_matching': True,
4628     }, {
4629         'note': 'basic, single video playlist',
4630         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4631         'info_dict': {
4632             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4633             'uploader': 'Sergey M.',
4634             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4635             'title': 'youtube-dl public playlist',
4636             'description': '',
4637             'tags': [],
4638             'view_count': int,
4639             'modified_date': '20201130',
4640             'channel': 'Sergey M.',
4641             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4642             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4643             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4644         },
4645         'playlist_count': 1,
4646     }, {
4647         'note': 'empty playlist',
4648         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4649         'info_dict': {
4650             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4651             'uploader': 'Sergey M.',
4652             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4653             'title': 'youtube-dl empty playlist',
4654             'tags': [],
4655             'channel': 'Sergey M.',
4656             'description': '',
4657             'modified_date': '20160902',
4658             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4659             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4660             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4661         },
4662         'playlist_count': 0,
4663     }, {
4664         'note': 'Home tab',
4665         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4666         'info_dict': {
4667             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4668             'title': 'lex will - Home',
4669             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4670             'uploader': 'lex will',
4671             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4672             'channel': 'lex will',
4673             'tags': ['bible', 'history', 'prophesy'],
4674             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4675             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4676             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4677             'channel_follower_count': int
4678         },
4679         'playlist_mincount': 2,
4680     }, {
4681         'note': 'Videos tab',
4682         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4683         'info_dict': {
4684             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4685             'title': 'lex will - Videos',
4686             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4687             'uploader': 'lex will',
4688             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4689             'tags': ['bible', 'history', 'prophesy'],
4690             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4691             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4692             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4693             'channel': 'lex will',
4694             'channel_follower_count': int
4695         },
4696         'playlist_mincount': 975,
4697     }, {
4698         'note': 'Videos tab, sorted by popular',
4699         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4700         'info_dict': {
4701             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4702             'title': 'lex will - Videos',
4703             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4704             'uploader': 'lex will',
4705             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4706             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4707             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4708             'channel': 'lex will',
4709             'tags': ['bible', 'history', 'prophesy'],
4710             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4711             'channel_follower_count': int
4712         },
4713         'playlist_mincount': 199,
4714     }, {
4715         'note': 'Playlists tab',
4716         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4717         'info_dict': {
4718             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4719             'title': 'lex will - Playlists',
4720             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4721             'uploader': 'lex will',
4722             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4723             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4724             'channel': 'lex will',
4725             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4726             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4727             'tags': ['bible', 'history', 'prophesy'],
4728             'channel_follower_count': int
4729         },
4730         'playlist_mincount': 17,
4731     }, {
4732         'note': 'Community tab',
4733         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4734         'info_dict': {
4735             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4736             'title': 'lex will - Community',
4737             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4738             'uploader': 'lex will',
4739             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4740             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4741             'channel': 'lex will',
4742             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4743             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4744             'tags': ['bible', 'history', 'prophesy'],
4745             'channel_follower_count': int
4746         },
4747         'playlist_mincount': 18,
4748     }, {
4749         'note': 'Channels tab',
4750         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4751         'info_dict': {
4752             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4753             'title': 'lex will - Channels',
4754             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4755             'uploader': 'lex will',
4756             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4757             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4758             'channel': 'lex will',
4759             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4760             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4761             'tags': ['bible', 'history', 'prophesy'],
4762             'channel_follower_count': int
4763         },
4764         'playlist_mincount': 12,
4765     }, {
4766         'note': 'Search tab',
4767         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4768         'playlist_mincount': 40,
4769         'info_dict': {
4770             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4771             'title': '3Blue1Brown - Search - linear algebra',
4772             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4773             'uploader': '3Blue1Brown',
4774             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4775             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4776             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4777             'tags': ['Mathematics'],
4778             'channel': '3Blue1Brown',
4779             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4780             'channel_follower_count': int
4781         },
4782     }, {
4783         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4784         'only_matching': True,
4785     }, {
4786         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4787         'only_matching': True,
4788     }, {
4789         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4790         'only_matching': True,
4791     }, {
4792         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4793         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4794         'info_dict': {
4795             'title': '29C3: Not my department',
4796             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4797             'uploader': 'Christiaan008',
4798             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4799             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4800             'tags': [],
4801             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4802             'view_count': int,
4803             'modified_date': '20150605',
4804             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4805             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4806             'channel': 'Christiaan008',
4807         },
4808         'playlist_count': 96,
4809     }, {
4810         'note': 'Large playlist',
4811         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4812         'info_dict': {
4813             'title': 'Uploads from Cauchemar',
4814             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4815             'uploader': 'Cauchemar',
4816             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4817             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4818             'tags': [],
4819             'modified_date': r're:\d{8}',
4820             'channel': 'Cauchemar',
4821             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4822             'view_count': int,
4823             'description': '',
4824             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4825         },
4826         'playlist_mincount': 1123,
4827         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4828     }, {
4829         'note': 'even larger playlist, 8832 videos',
4830         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4831         'only_matching': True,
4832     }, {
4833         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4834         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4835         'info_dict': {
4836             'title': 'Uploads from Interstellar Movie',
4837             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4838             'uploader': 'Interstellar Movie',
4839             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4840             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4841             'tags': [],
4842             'view_count': int,
4843             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4844             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4845             'channel': 'Interstellar Movie',
4846             'description': '',
4847             'modified_date': r're:\d{8}',
4848         },
4849         'playlist_mincount': 21,
4850     }, {
4851         'note': 'Playlist with "show unavailable videos" button',
4852         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4853         'info_dict': {
4854             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4855             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4856             'uploader': 'Phim Siêu Nhân Nhật Bản',
4857             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4858             'view_count': int,
4859             'channel': 'Phim Siêu Nhân Nhật Bản',
4860             'tags': [],
4861             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4862             'description': '',
4863             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4864             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4865             'modified_date': r're:\d{8}',
4866         },
4867         'playlist_mincount': 200,
4868         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4869     }, {
4870         'note': 'Playlist with unavailable videos in page 7',
4871         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4872         'info_dict': {
4873             'title': 'Uploads from BlankTV',
4874             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4875             'uploader': 'BlankTV',
4876             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4877             'channel': 'BlankTV',
4878             'channel_url': 'https://www.youtube.com/c/blanktv',
4879             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4880             'view_count': int,
4881             'tags': [],
4882             'uploader_url': 'https://www.youtube.com/c/blanktv',
4883             'modified_date': r're:\d{8}',
4884             'description': '',
4885         },
4886         'playlist_mincount': 1000,
4887         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4888     }, {
4889         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4890         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4891         'info_dict': {
4892             'title': 'Data Analysis with Dr Mike Pound',
4893             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4894             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4895             'uploader': 'Computerphile',
4896             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4897             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4898             'tags': [],
4899             'view_count': int,
4900             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4901             'channel_url': 'https://www.youtube.com/user/Computerphile',
4902             'channel': 'Computerphile',
4903         },
4904         'playlist_mincount': 11,
4905     }, {
4906         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4907         'only_matching': True,
4908     }, {
4909         'note': 'Playlist URL that does not actually serve a playlist',
4910         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4911         'info_dict': {
4912             'id': 'FqZTN594JQw',
4913             'ext': 'webm',
4914             'title': "Smiley's People 01 detective, Adventure Series, Action",
4915             'uploader': 'STREEM',
4916             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4917             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4918             'upload_date': '20150526',
4919             'license': 'Standard YouTube License',
4920             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4921             'categories': ['People & Blogs'],
4922             'tags': list,
4923             'view_count': int,
4924             'like_count': int,
4925         },
4926         'params': {
4927             'skip_download': True,
4928         },
4929         'skip': 'This video is not available.',
4930         'add_ie': [YoutubeIE.ie_key()],
4931     }, {
4932         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
4933         'only_matching': True,
4934     }, {
4935         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
4936         'only_matching': True,
4937     }, {
4938         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
4939         'info_dict': {
4940             'id': 'GgL890LIznQ',  # This will keep changing
4941             'ext': 'mp4',
4942             'title': str,
4943             'uploader': 'Sky News',
4944             'uploader_id': 'skynews',
4945             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
4946             'upload_date': r're:\d{8}',
4947             'description': str,
4948             'categories': ['News & Politics'],
4949             'tags': list,
4950             'like_count': int,
4951             'release_timestamp': 1642502819,
4952             'channel': 'Sky News',
4953             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
4954             'age_limit': 0,
4955             'view_count': int,
4956             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
4957             'playable_in_embed': True,
4958             'release_date': '20220118',
4959             'availability': 'public',
4960             'live_status': 'is_live',
4961             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
4962             'channel_follower_count': int
4963         },
4964         'params': {
4965             'skip_download': True,
4966         },
4967         'expected_warnings': ['Ignoring subtitle tracks found in '],
4968     }, {
4969         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
4970         'info_dict': {
4971             'id': 'a48o2S1cPoo',
4972             'ext': 'mp4',
4973             'title': 'The Young Turks - Live Main Show',
4974             'uploader': 'The Young Turks',
4975             'uploader_id': 'TheYoungTurks',
4976             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
4977             'upload_date': '20150715',
4978             'license': 'Standard YouTube License',
4979             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
4980             'categories': ['News & Politics'],
4981             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
4982             'like_count': int,
4983         },
4984         'params': {
4985             'skip_download': True,
4986         },
4987         'only_matching': True,
4988     }, {
4989         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
4990         'only_matching': True,
4991     }, {
4992         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
4993         'only_matching': True,
4994     }, {
4995         'note': 'A channel that is not live. Should raise error',
4996         'url': 'https://www.youtube.com/user/numberphile/live',
4997         'only_matching': True,
4998     }, {
4999         'url': 'https://www.youtube.com/feed/trending',
5000         'only_matching': True,
5001     }, {
5002         'url': 'https://www.youtube.com/feed/library',
5003         'only_matching': True,
5004     }, {
5005         'url': 'https://www.youtube.com/feed/history',
5006         'only_matching': True,
5007     }, {
5008         'url': 'https://www.youtube.com/feed/subscriptions',
5009         'only_matching': True,
5010     }, {
5011         'url': 'https://www.youtube.com/feed/watch_later',
5012         'only_matching': True,
5013     }, {
5014         'note': 'Recommended - redirects to home page.',
5015         'url': 'https://www.youtube.com/feed/recommended',
5016         'only_matching': True,
5017     }, {
5018         'note': 'inline playlist with not always working continuations',
5019         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5020         'only_matching': True,
5021     }, {
5022         'url': 'https://www.youtube.com/course',
5023         'only_matching': True,
5024     }, {
5025         'url': 'https://www.youtube.com/zsecurity',
5026         'only_matching': True,
5027     }, {
5028         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5029         'only_matching': True,
5030     }, {
5031         'url': 'https://www.youtube.com/TheYoungTurks/live',
5032         'only_matching': True,
5033     }, {
5034         'url': 'https://www.youtube.com/hashtag/cctv9',
5035         'info_dict': {
5036             'id': 'cctv9',
5037             'title': '#cctv9',
5038             'tags': [],
5039         },
5040         'playlist_mincount': 350,
5041     }, {
5042         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5043         'only_matching': True,
5044     }, {
5045         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5046         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5047         'only_matching': True
5048     }, {
5049         'note': '/browse/ should redirect to /channel/',
5050         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5051         'only_matching': True
5052     }, {
5053         'note': 'VLPL, should redirect to playlist?list=PL...',
5054         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5055         'info_dict': {
5056             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5057             'uploader': 'NoCopyrightSounds',
5058             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5059             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5060             'title': 'NCS Releases',
5061             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5062             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5063             'modified_date': r're:\d{8}',
5064             'view_count': int,
5065             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5066             'tags': [],
5067             'channel': 'NoCopyrightSounds',
5068         },
5069         'playlist_mincount': 166,
5070         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5071     }, {
5072         'note': 'Topic, should redirect to playlist?list=UU...',
5073         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5074         'info_dict': {
5075             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5076             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5077             'title': 'Uploads from Royalty Free Music - Topic',
5078             'uploader': 'Royalty Free Music - Topic',
5079             'tags': [],
5080             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5081             'channel': 'Royalty Free Music - Topic',
5082             'view_count': int,
5083             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5084             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5085             'modified_date': r're:\d{8}',
5086             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5087             'description': '',
5088         },
5089         'expected_warnings': [
5090             'The URL does not have a videos tab',
5091             r'[Uu]navailable videos (are|will be) hidden',
5092         ],
5093         'playlist_mincount': 101,
5094     }, {
5095         'note': 'Topic without a UU playlist',
5096         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5097         'info_dict': {
5098             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5099             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5100             'tags': [],
5101         },
5102         'expected_warnings': [
5103             'the playlist redirect gave error',
5104         ],
5105         'playlist_mincount': 9,
5106     }, {
5107         'note': 'Youtube music Album',
5108         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5109         'info_dict': {
5110             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5111             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5112             'tags': [],
5113             'view_count': int,
5114             'description': '',
5115             'availability': 'unlisted',
5116             'modified_date': r're:\d{8}',
5117         },
5118         'playlist_count': 50,
5119     }, {
5120         'note': 'unlisted single video playlist',
5121         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5122         'info_dict': {
5123             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5124             'uploader': 'colethedj',
5125             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5126             'title': 'yt-dlp unlisted playlist test',
5127             'availability': 'unlisted',
5128             'tags': [],
5129             'modified_date': '20211208',
5130             'channel': 'colethedj',
5131             'view_count': int,
5132             'description': '',
5133             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5134             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5135             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5136         },
5137         'playlist_count': 1,
5138     }, {
5139         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5140         'url': 'https://www.youtube.com/feed/recommended',
5141         'info_dict': {
5142             'id': 'recommended',
5143             'title': 'recommended',
5144             'tags': [],
5145         },
5146         'playlist_mincount': 50,
5147         'params': {
5148             'skip_download': True,
5149             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5150         },
5151     }, {
5152         'note': 'API Fallback: /videos tab, sorted by oldest first',
5153         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5154         'info_dict': {
5155             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5156             'title': 'Cody\'sLab - Videos',
5157             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5158             'uploader': 'Cody\'sLab',
5159             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5160             'channel': 'Cody\'sLab',
5161             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5162             'tags': [],
5163             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5164             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5165             'channel_follower_count': int
5166         },
5167         'playlist_mincount': 650,
5168         'params': {
5169             'skip_download': True,
5170             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5171         },
5172     }, {
5173         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5174         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5175         'info_dict': {
5176             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5177             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5178             'title': 'Uploads from Royalty Free Music - Topic',
5179             'uploader': 'Royalty Free Music - Topic',
5180             'modified_date': r're:\d{8}',
5181             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5182             'description': '',
5183             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5184             'tags': [],
5185             'channel': 'Royalty Free Music - Topic',
5186             'view_count': int,
5187             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5188         },
5189         'expected_warnings': [
5190             'does not have a videos tab',
5191             r'[Uu]navailable videos (are|will be) hidden',
5192         ],
5193         'playlist_mincount': 101,
5194         'params': {
5195             'skip_download': True,
5196             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5197         },
5198     }, {
5199         'note': 'non-standard redirect to regional channel',
5200         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5201         'only_matching': True
5202     }, {
5203         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5204         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5205         'info_dict': {
5206             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5207             'modified_date': '20220407',
5208             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5209             'tags': [],
5210             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5211             'uploader': 'pukkandan',
5212             'availability': 'unlisted',
5213             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5214             'channel': 'pukkandan',
5215             'description': 'Test for collaborative playlist',
5216             'title': 'yt-dlp test - collaborative playlist',
5217             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5218         },
5219         'playlist_mincount': 2
5220     }]
5221
5222     @classmethod
5223     def suitable(cls, url):
5224         return False if YoutubeIE.suitable(url) else super().suitable(url)
5225
5226     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5227
5228     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5229     def _real_extract(self, url, smuggled_data):
5230         item_id = self._match_id(url)
5231         url = compat_urlparse.urlunparse(
5232             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
5233         compat_opts = self.get_param('compat_opts', [])
5234
5235         def get_mobj(url):
5236             mobj = self._URL_RE.match(url).groupdict()
5237             mobj.update((k, '') for k, v in mobj.items() if v is None)
5238             return mobj
5239
5240         mobj, redirect_warning = get_mobj(url), None
5241         # Youtube returns incomplete data if tabname is not lower case
5242         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5243         if is_channel:
5244             if smuggled_data.get('is_music_url'):
5245                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5246                     item_id = item_id[2:]
5247                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5248                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5249                     mdata = self._extract_tab_endpoint(
5250                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5251                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5252                                         get_all=False, expected_type=compat_str)
5253                     if not murl:
5254                         raise ExtractorError('Failed to resolve album to playlist')
5255                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5256                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5257                     pre = f'https://www.youtube.com/channel/{item_id}'
5258
5259         original_tab_name = tab
5260         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5261             # Home URLs should redirect to /videos/
5262             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5263                                 'To download only the videos in the home page, add a "/featured" to the URL')
5264             tab = '/videos'
5265
5266         url = ''.join((pre, tab, post))
5267         mobj = get_mobj(url)
5268
5269         # Handle both video/playlist URLs
5270         qs = parse_qs(url)
5271         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5272
5273         if not video_id and mobj['not_channel'].startswith('watch'):
5274             if not playlist_id:
5275                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5276                 raise ExtractorError('Unable to recognize tab page')
5277             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5278             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5279             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5280             mobj = get_mobj(url)
5281
5282         if video_id and playlist_id:
5283             if self.get_param('noplaylist'):
5284                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5285                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5286                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5287             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5288
5289         data, ytcfg = self._extract_data(url, item_id)
5290
5291         # YouTube may provide a non-standard redirect to the regional channel
5292         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5293         redirect_url = traverse_obj(
5294             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5295         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5296             redirect_url = ''.join((
5297                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5298             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5299             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5300
5301         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5302         if tabs:
5303             selected_tab = self._extract_selected_tab(tabs)
5304             selected_tab_name = selected_tab.get('title', '').lower()
5305             if selected_tab_name == 'home':
5306                 selected_tab_name = 'featured'
5307             requested_tab_name = mobj['tab'][1:]
5308             if 'no-youtube-channel-redirect' not in compat_opts:
5309                 if requested_tab_name == 'live':
5310                     # Live tab should have redirected to the video
5311                     raise ExtractorError('The channel is not currently live', expected=True)
5312                 if requested_tab_name not in ('', selected_tab_name):
5313                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5314                     if not original_tab_name:
5315                         if item_id[:2] == 'UC':
5316                             # Topic channels don't have /videos. Use the equivalent playlist instead
5317                             pl_id = f'UU{item_id[2:]}'
5318                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5319                             try:
5320                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5321                             except ExtractorError:
5322                                 redirect_warning += ' and the playlist redirect gave error'
5323                             else:
5324                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5325                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5326                         if selected_tab_name and selected_tab_name != requested_tab_name:
5327                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5328                     else:
5329                         raise ExtractorError(redirect_warning, expected=True)
5330
5331         if redirect_warning:
5332             self.to_screen(redirect_warning)
5333         self.write_debug(f'Final URL: {url}')
5334
5335         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5336         if 'no-youtube-unavailable-videos' not in compat_opts:
5337             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5338         self._extract_and_report_alerts(data, only_once=True)
5339         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5340         if tabs:
5341             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5342
5343         playlist = traverse_obj(
5344             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5345         if playlist:
5346             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5347
5348         video_id = traverse_obj(
5349             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5350         if video_id:
5351             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5352                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5353             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5354                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5355
5356         raise ExtractorError('Unable to recognize tab page')
5357
5358
5359 class YoutubePlaylistIE(InfoExtractor):
5360     IE_DESC = 'YouTube playlists'
5361     _VALID_URL = r'''(?x)(?:
5362                         (?:https?://)?
5363                         (?:\w+\.)?
5364                         (?:
5365                             (?:
5366                                 youtube(?:kids)?\.com|
5367                                 %(invidious)s
5368                             )
5369                             /.*?\?.*?\blist=
5370                         )?
5371                         (?P<id>%(playlist_id)s)
5372                      )''' % {
5373         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5374         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5375     }
5376     IE_NAME = 'youtube:playlist'
5377     _TESTS = [{
5378         'note': 'issue #673',
5379         'url': 'PLBB231211A4F62143',
5380         'info_dict': {
5381             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5382             'id': 'PLBB231211A4F62143',
5383             'uploader': 'Wickman',
5384             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5385             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5386             'view_count': int,
5387             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5388             'modified_date': r're:\d{8}',
5389             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5390             'channel': 'Wickman',
5391             'tags': [],
5392             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5393         },
5394         'playlist_mincount': 29,
5395     }, {
5396         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5397         'info_dict': {
5398             'title': 'YDL_safe_search',
5399             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5400         },
5401         'playlist_count': 2,
5402         'skip': 'This playlist is private',
5403     }, {
5404         'note': 'embedded',
5405         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5406         'playlist_count': 4,
5407         'info_dict': {
5408             'title': 'JODA15',
5409             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5410             'uploader': 'milan',
5411             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5412             'description': '',
5413             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5414             'tags': [],
5415             'modified_date': '20140919',
5416             'view_count': int,
5417             'channel': 'milan',
5418             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5419             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5420         },
5421         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5422     }, {
5423         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5424         'playlist_mincount': 654,
5425         'info_dict': {
5426             'title': '2018 Chinese New Singles (11/6 updated)',
5427             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5428             'uploader': 'LBK',
5429             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5430             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5431             'channel': 'LBK',
5432             'view_count': int,
5433             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5434             'tags': [],
5435             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5436             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5437             'modified_date': r're:\d{8}',
5438         },
5439         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5440     }, {
5441         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5442         'only_matching': True,
5443     }, {
5444         # music album playlist
5445         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5446         'only_matching': True,
5447     }]
5448
5449     @classmethod
5450     def suitable(cls, url):
5451         if YoutubeTabIE.suitable(url):
5452             return False
5453         from ..utils import parse_qs
5454         qs = parse_qs(url)
5455         if qs.get('v', [None])[0]:
5456             return False
5457         return super().suitable(url)
5458
5459     def _real_extract(self, url):
5460         playlist_id = self._match_id(url)
5461         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5462         url = update_url_query(
5463             'https://www.youtube.com/playlist',
5464             parse_qs(url) or {'list': playlist_id})
5465         if is_music_url:
5466             url = smuggle_url(url, {'is_music_url': True})
5467         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5468
5469
5470 class YoutubeYtBeIE(InfoExtractor):
5471     IE_DESC = 'youtu.be'
5472     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5473     _TESTS = [{
5474         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5475         'info_dict': {
5476             'id': 'yeWKywCrFtk',
5477             'ext': 'mp4',
5478             'title': 'Small Scale Baler and Braiding Rugs',
5479             'uploader': 'Backus-Page House Museum',
5480             'uploader_id': 'backuspagemuseum',
5481             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5482             'upload_date': '20161008',
5483             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5484             'categories': ['Nonprofits & Activism'],
5485             'tags': list,
5486             'like_count': int,
5487             'age_limit': 0,
5488             'playable_in_embed': True,
5489             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5490             'channel': 'Backus-Page House Museum',
5491             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5492             'live_status': 'not_live',
5493             'view_count': int,
5494             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5495             'availability': 'public',
5496             'duration': 59,
5497         },
5498         'params': {
5499             'noplaylist': True,
5500             'skip_download': True,
5501         },
5502     }, {
5503         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5504         'only_matching': True,
5505     }]
5506
5507     def _real_extract(self, url):
5508         mobj = self._match_valid_url(url)
5509         video_id = mobj.group('id')
5510         playlist_id = mobj.group('playlist_id')
5511         return self.url_result(
5512             update_url_query('https://www.youtube.com/watch', {
5513                 'v': video_id,
5514                 'list': playlist_id,
5515                 'feature': 'youtu.be',
5516             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5517
5518
5519 class YoutubeLivestreamEmbedIE(InfoExtractor):
5520     IE_DESC = 'YouTube livestream embeds'
5521     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5522     _TESTS = [{
5523         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5524         'only_matching': True,
5525     }]
5526
5527     def _real_extract(self, url):
5528         channel_id = self._match_id(url)
5529         return self.url_result(
5530             f'https://www.youtube.com/channel/{channel_id}/live',
5531             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5532
5533
5534 class YoutubeYtUserIE(InfoExtractor):
5535     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5536     IE_NAME = 'youtube:user'
5537     _VALID_URL = r'ytuser:(?P<id>.+)'
5538     _TESTS = [{
5539         'url': 'ytuser:phihag',
5540         'only_matching': True,
5541     }]
5542
5543     def _real_extract(self, url):
5544         user_id = self._match_id(url)
5545         return self.url_result(
5546             'https://www.youtube.com/user/%s/videos' % user_id,
5547             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5548
5549
5550 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5551     IE_NAME = 'youtube:favorites'
5552     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5553     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5554     _LOGIN_REQUIRED = True
5555     _TESTS = [{
5556         'url': ':ytfav',
5557         'only_matching': True,
5558     }, {
5559         'url': ':ytfavorites',
5560         'only_matching': True,
5561     }]
5562
5563     def _real_extract(self, url):
5564         return self.url_result(
5565             'https://www.youtube.com/playlist?list=LL',
5566             ie=YoutubeTabIE.ie_key())
5567
5568
5569 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5570     IE_NAME = 'youtube:notif'
5571     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5572     _VALID_URL = r':ytnotif(?:ication)?s?'
5573     _LOGIN_REQUIRED = True
5574     _TESTS = [{
5575         'url': ':ytnotif',
5576         'only_matching': True,
5577     }, {
5578         'url': ':ytnotifications',
5579         'only_matching': True,
5580     }]
5581
5582     def _extract_notification_menu(self, response, continuation_list):
5583         notification_list = traverse_obj(
5584             response,
5585             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5586             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5587             expected_type=list) or []
5588         continuation_list[0] = None
5589         for item in notification_list:
5590             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5591             if entry:
5592                 yield entry
5593             continuation = item.get('continuationItemRenderer')
5594             if continuation:
5595                 continuation_list[0] = continuation
5596
5597     def _extract_notification_renderer(self, notification):
5598         video_id = traverse_obj(
5599             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5600         url = f'https://www.youtube.com/watch?v={video_id}'
5601         channel_id = None
5602         if not video_id:
5603             browse_ep = traverse_obj(
5604                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5605             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5606             post_id = self._search_regex(
5607                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5608                 'post id', default=None)
5609             if not channel_id or not post_id:
5610                 return
5611             # The direct /post url redirects to this in the browser
5612             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5613
5614         channel = traverse_obj(
5615             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5616             expected_type=str)
5617         title = self._search_regex(
5618             rf'{re.escape(channel)} [^:]+: (.+)', self._get_text(notification, 'shortMessage'),
5619             'video title', default=None)
5620         if title:
5621             title = title.replace('\xad', '')  # remove soft hyphens
5622         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5623                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5624                        else None)
5625         return {
5626             '_type': 'url',
5627             'url': url,
5628             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5629             'video_id': video_id,
5630             'title': title,
5631             'channel_id': channel_id,
5632             'channel': channel,
5633             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5634             'upload_date': upload_date,
5635         }
5636
5637     def _notification_menu_entries(self, ytcfg):
5638         continuation_list = [None]
5639         response = None
5640         for page in itertools.count(1):
5641             ctoken = traverse_obj(
5642                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5643             response = self._extract_response(
5644                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5645                 ep='notification/get_notification_menu', check_get_keys='actions',
5646                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5647             yield from self._extract_notification_menu(response, continuation_list)
5648             if not continuation_list[0]:
5649                 break
5650
5651     def _real_extract(self, url):
5652         display_id = 'notifications'
5653         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5654         self._report_playlist_authcheck(ytcfg)
5655         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5656
5657
5658 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5659     IE_DESC = 'YouTube search'
5660     IE_NAME = 'youtube:search'
5661     _SEARCH_KEY = 'ytsearch'
5662     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5663     _TESTS = [{
5664         'url': 'ytsearch5:youtube-dl test video',
5665         'playlist_count': 5,
5666         'info_dict': {
5667             'id': 'youtube-dl test video',
5668             'title': 'youtube-dl test video',
5669         }
5670     }]
5671
5672
5673 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5674     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5675     _SEARCH_KEY = 'ytsearchdate'
5676     IE_DESC = 'YouTube search, newest videos first'
5677     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5678     _TESTS = [{
5679         'url': 'ytsearchdate5:youtube-dl test video',
5680         'playlist_count': 5,
5681         'info_dict': {
5682             'id': 'youtube-dl test video',
5683             'title': 'youtube-dl test video',
5684         }
5685     }]
5686
5687
5688 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5689     IE_DESC = 'YouTube search URLs with sorting and filter support'
5690     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5691     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5692     _TESTS = [{
5693         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5694         'playlist_mincount': 5,
5695         'info_dict': {
5696             'id': 'youtube-dl test video',
5697             'title': 'youtube-dl test video',
5698         }
5699     }, {
5700         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5701         'playlist_mincount': 5,
5702         'info_dict': {
5703             'id': 'python',
5704             'title': 'python',
5705         }
5706     }, {
5707         'url': 'https://www.youtube.com/results?search_query=%23cats',
5708         'playlist_mincount': 1,
5709         'info_dict': {
5710             'id': '#cats',
5711             'title': '#cats',
5712             'entries': [{
5713                 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5714                 'title': '#cats',
5715             }],
5716         },
5717     }, {
5718         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5719         'only_matching': True,
5720     }]
5721
5722     def _real_extract(self, url):
5723         qs = parse_qs(url)
5724         query = (qs.get('search_query') or qs.get('q'))[0]
5725         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5726
5727
5728 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5729     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5730     IE_NAME = 'youtube:music:search_url'
5731     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5732     _TESTS = [{
5733         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5734         'playlist_count': 16,
5735         'info_dict': {
5736             'id': 'royalty free music',
5737             'title': 'royalty free music',
5738         }
5739     }, {
5740         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5741         'playlist_mincount': 30,
5742         'info_dict': {
5743             'id': 'royalty free music - songs',
5744             'title': 'royalty free music - songs',
5745         },
5746         'params': {'extract_flat': 'in_playlist'}
5747     }, {
5748         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5749         'playlist_mincount': 30,
5750         'info_dict': {
5751             'id': 'royalty free music - community playlists',
5752             'title': 'royalty free music - community playlists',
5753         },
5754         'params': {'extract_flat': 'in_playlist'}
5755     }]
5756
5757     _SECTIONS = {
5758         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5759         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5760         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5761         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5762         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5763         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5764     }
5765
5766     def _real_extract(self, url):
5767         qs = parse_qs(url)
5768         query = (qs.get('search_query') or qs.get('q'))[0]
5769         params = qs.get('sp', (None,))[0]
5770         if params:
5771             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5772         else:
5773             section = compat_urllib_parse_unquote_plus((url.split('#') + [''])[1]).lower()
5774             params = self._SECTIONS.get(section)
5775             if not params:
5776                 section = None
5777         title = join_nonempty(query, section, delim=' - ')
5778         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5779
5780
5781 class YoutubeFeedsInfoExtractor(InfoExtractor):
5782     """
5783     Base class for feed extractors
5784     Subclasses must define the _FEED_NAME property.
5785     """
5786     _LOGIN_REQUIRED = True
5787
5788     def _real_initialize(self):
5789         YoutubeBaseInfoExtractor._check_login_required(self)
5790
5791     @property
5792     def IE_NAME(self):
5793         return 'youtube:%s' % self._FEED_NAME
5794
5795     def _real_extract(self, url):
5796         return self.url_result(
5797             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5798
5799
5800 class YoutubeWatchLaterIE(InfoExtractor):
5801     IE_NAME = 'youtube:watchlater'
5802     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5803     _VALID_URL = r':ytwatchlater'
5804     _TESTS = [{
5805         'url': ':ytwatchlater',
5806         'only_matching': True,
5807     }]
5808
5809     def _real_extract(self, url):
5810         return self.url_result(
5811             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5812
5813
5814 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5815     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5816     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5817     _FEED_NAME = 'recommended'
5818     _LOGIN_REQUIRED = False
5819     _TESTS = [{
5820         'url': ':ytrec',
5821         'only_matching': True,
5822     }, {
5823         'url': ':ytrecommended',
5824         'only_matching': True,
5825     }, {
5826         'url': 'https://youtube.com',
5827         'only_matching': True,
5828     }]
5829
5830
5831 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5832     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5833     _VALID_URL = r':ytsub(?:scription)?s?'
5834     _FEED_NAME = 'subscriptions'
5835     _TESTS = [{
5836         'url': ':ytsubs',
5837         'only_matching': True,
5838     }, {
5839         'url': ':ytsubscriptions',
5840         'only_matching': True,
5841     }]
5842
5843
5844 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5845     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5846     _VALID_URL = r':ythis(?:tory)?'
5847     _FEED_NAME = 'history'
5848     _TESTS = [{
5849         'url': ':ythistory',
5850         'only_matching': True,
5851     }]
5852
5853
5854 class YoutubeStoriesIE(InfoExtractor):
5855     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5856     IE_NAME = 'youtube:stories'
5857     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5858     _TESTS = [{
5859         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5860         'only_matching': True,
5861     }]
5862
5863     def _real_extract(self, url):
5864         playlist_id = f'RLTD{self._match_id(url)}'
5865         return self.url_result(
5866             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5867             ie=YoutubeTabIE, video_id=playlist_id)
5868
5869
5870 class YoutubeTruncatedURLIE(InfoExtractor):
5871     IE_NAME = 'youtube:truncated_url'
5872     IE_DESC = False  # Do not list
5873     _VALID_URL = r'''(?x)
5874         (?:https?://)?
5875         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5876         (?:watch\?(?:
5877             feature=[a-z_]+|
5878             annotation_id=annotation_[^&]+|
5879             x-yt-cl=[0-9]+|
5880             hl=[^&]*|
5881             t=[0-9]+
5882         )?
5883         |
5884             attribution_link\?a=[^&]+
5885         )
5886         $
5887     '''
5888
5889     _TESTS = [{
5890         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5891         'only_matching': True,
5892     }, {
5893         'url': 'https://www.youtube.com/watch?',
5894         'only_matching': True,
5895     }, {
5896         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5897         'only_matching': True,
5898     }, {
5899         'url': 'https://www.youtube.com/watch?feature=foo',
5900         'only_matching': True,
5901     }, {
5902         'url': 'https://www.youtube.com/watch?hl=en-GB',
5903         'only_matching': True,
5904     }, {
5905         'url': 'https://www.youtube.com/watch?t=2372',
5906         'only_matching': True,
5907     }]
5908
5909     def _real_extract(self, url):
5910         raise ExtractorError(
5911             'Did you forget to quote the URL? Remember that & is a meta '
5912             'character in most shells, so you want to put the URL in quotes, '
5913             'like  youtube-dl '
5914             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5915             ' or simply  youtube-dl BaW_jenozKc  .',
5916             expected=True)
5917
5918
5919 class YoutubeClipIE(InfoExtractor):
5920     IE_NAME = 'youtube:clip'
5921     IE_DESC = False  # Do not list
5922     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/'
5923
5924     def _real_extract(self, url):
5925         self.report_warning('YouTube clips are not currently supported. The entire video will be downloaded instead')
5926         return self.url_result(url, 'Generic')
5927
5928
5929 class YoutubeTruncatedIDIE(InfoExtractor):
5930     IE_NAME = 'youtube:truncated_id'
5931     IE_DESC = False  # Do not list
5932     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
5933
5934     _TESTS = [{
5935         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
5936         'only_matching': True,
5937     }]
5938
5939     def _real_extract(self, url):
5940         video_id = self._match_id(url)
5941         raise ExtractorError(
5942             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
5943             expected=True)