yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import functools
   6 import hashlib
   7 import itertools
   8 import json
   9 import math
  10 import os.path
  11 import random
  12 import re
  13 import sys
  14 import threading
  15 import time
  16 import traceback
  17
  18 from .common import InfoExtractor, SearchInfoExtractor
  19 from ..compat import (
  20     compat_chr,
  21     compat_HTTPError,
  22     compat_parse_qs,
  23     compat_str,
  24     compat_urllib_parse_unquote_plus,
  25     compat_urllib_parse_urlencode,
  26     compat_urllib_parse_urlparse,
  27     compat_urlparse,
  28 )
  29 from ..jsinterp import JSInterpreter
  30 from ..utils import (
  31     NO_DEFAULT,
  32     ExtractorError,
  33     bug_reports_message,
  34     classproperty,
  35     clean_html,
  36     datetime_from_str,
  37     dict_get,
  38     error_to_compat_str,
  39     float_or_none,
  40     format_field,
  41     get_first,
  42     int_or_none,
  43     is_html,
  44     join_nonempty,
  45     js_to_json,
  46     mimetype2ext,
  47     network_exceptions,
  48     orderedSet,
  49     parse_codecs,
  50     parse_count,
  51     parse_duration,
  52     parse_iso8601,
  53     parse_qs,
  54     qualities,
  55     remove_end,
  56     remove_start,
  57     smuggle_url,
  58     str_or_none,
  59     str_to_int,
  60     strftime_or_none,
  61     traverse_obj,
  62     try_get,
  63     unescapeHTML,
  64     unified_strdate,
  65     unified_timestamp,
  66     unsmuggle_url,
  67     update_url_query,
  68     url_or_none,
  69     urljoin,
  70     variadic,
  71 )
  72
  73 # any clients starting with _ cannot be explicity requested by the user
  74 INNERTUBE_CLIENTS = {
  75     'web': {
  76         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  77         'INNERTUBE_CONTEXT': {
  78             'client': {
  79                 'clientName': 'WEB',
  80                 'clientVersion': '2.20211221.00.00',
  81             }
  82         },
  83         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  84     },
  85     'web_embedded': {
  86         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  87         'INNERTUBE_CONTEXT': {
  88             'client': {
  89                 'clientName': 'WEB_EMBEDDED_PLAYER',
  90                 'clientVersion': '1.20211215.00.01',
  91             },
  92         },
  93         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  94     },
  95     'web_music': {
  96         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  97         'INNERTUBE_HOST': 'music.youtube.com',
  98         'INNERTUBE_CONTEXT': {
  99             'client': {
 100                 'clientName': 'WEB_REMIX',
 101                 'clientVersion': '1.20211213.00.00',
 102             }
 103         },
 104         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 105     },
 106     'web_creator': {
 107         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 108         'INNERTUBE_CONTEXT': {
 109             'client': {
 110                 'clientName': 'WEB_CREATOR',
 111                 'clientVersion': '1.20211220.02.00',
 112             }
 113         },
 114         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 115     },
 116     'android': {
 117         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 118         'INNERTUBE_CONTEXT': {
 119             'client': {
 120                 'clientName': 'ANDROID',
 121                 'clientVersion': '16.49',
 122             }
 123         },
 124         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 125         'REQUIRE_JS_PLAYER': False
 126     },
 127     'android_embedded': {
 128         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 129         'INNERTUBE_CONTEXT': {
 130             'client': {
 131                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 132                 'clientVersion': '16.49',
 133             },
 134         },
 135         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 136         'REQUIRE_JS_PLAYER': False
 137     },
 138     'android_music': {
 139         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 140         'INNERTUBE_CONTEXT': {
 141             'client': {
 142                 'clientName': 'ANDROID_MUSIC',
 143                 'clientVersion': '4.57',
 144             }
 145         },
 146         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 147         'REQUIRE_JS_PLAYER': False
 148     },
 149     'android_creator': {
 150         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 151         'INNERTUBE_CONTEXT': {
 152             'client': {
 153                 'clientName': 'ANDROID_CREATOR',
 154                 'clientVersion': '21.47',
 155             },
 156         },
 157         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 158         'REQUIRE_JS_PLAYER': False
 159     },
 160     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 161     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 162     'ios': {
 163         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 164         'INNERTUBE_CONTEXT': {
 165             'client': {
 166                 'clientName': 'IOS',
 167                 'clientVersion': '16.46',
 168                 'deviceModel': 'iPhone14,3',
 169             }
 170         },
 171         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 172         'REQUIRE_JS_PLAYER': False
 173     },
 174     'ios_embedded': {
 175         'INNERTUBE_CONTEXT': {
 176             'client': {
 177                 'clientName': 'IOS_MESSAGES_EXTENSION',
 178                 'clientVersion': '16.46',
 179                 'deviceModel': 'iPhone14,3',
 180             },
 181         },
 182         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 183         'REQUIRE_JS_PLAYER': False
 184     },
 185     'ios_music': {
 186         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 187         'INNERTUBE_CONTEXT': {
 188             'client': {
 189                 'clientName': 'IOS_MUSIC',
 190                 'clientVersion': '4.57',
 191             },
 192         },
 193         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 194         'REQUIRE_JS_PLAYER': False
 195     },
 196     'ios_creator': {
 197         'INNERTUBE_CONTEXT': {
 198             'client': {
 199                 'clientName': 'IOS_CREATOR',
 200                 'clientVersion': '21.47',
 201             },
 202         },
 203         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 204         'REQUIRE_JS_PLAYER': False
 205     },
 206     # mweb has 'ultralow' formats
 207     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 208     'mweb': {
 209         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 210         'INNERTUBE_CONTEXT': {
 211             'client': {
 212                 'clientName': 'MWEB',
 213                 'clientVersion': '2.20211221.01.00',
 214             }
 215         },
 216         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 217     },
 218     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 219     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 220     'tv_embedded': {
 221         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 222         'INNERTUBE_CONTEXT': {
 223             'client': {
 224                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 225                 'clientVersion': '2.0',
 226             },
 227         },
 228         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 229     },
 230 }
 231
 232
 233 def _split_innertube_client(client_name):
 234     variant, *base = client_name.rsplit('.', 1)
 235     if base:
 236         return variant, base[0], variant
 237     base, *variant = client_name.split('_', 1)
 238     return client_name, base, variant[0] if variant else None
 239
 240
 241 def build_innertube_clients():
 242     THIRD_PARTY = {
 243         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 244     }
 245     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 246     priority = qualities(BASE_CLIENTS[::-1])
 247
 248     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 249         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 250         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 251         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 252         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 253
 254         _, base_client, variant = _split_innertube_client(client)
 255         ytcfg['priority'] = 10 * priority(base_client)
 256
 257         if not variant:
 258             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 259             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 260             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 261             embedscreen['priority'] -= 3
 262         elif variant == 'embedded':
 263             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 264             ytcfg['priority'] -= 2
 265         else:
 266             ytcfg['priority'] -= 3
 267
 268
 269 build_innertube_clients()
 270
 271
 272 class YoutubeBaseInfoExtractor(InfoExtractor):
 273     """Provide base functions for Youtube extractors"""
 274
 275     _RESERVED_NAMES = (
 276         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 277         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 278         r'browse|oembed|get_video_info|iframe_api|s/player|'
 279         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 280
 281     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 282
 283     # _NETRC_MACHINE = 'youtube'
 284
 285     # If True it will raise an error if no login info is provided
 286     _LOGIN_REQUIRED = False
 287
 288     _INVIDIOUS_SITES = (
 289         # invidious-redirect websites
 290         r'(?:www\.)?redirect\.invidious\.io',
 291         r'(?:(?:www|dev)\.)?invidio\.us',
 292         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 293         r'(?:www\.)?invidious\.pussthecat\.org',
 294         r'(?:www\.)?invidious\.zee\.li',
 295         r'(?:www\.)?invidious\.ethibox\.fr',
 296         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 297         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 298         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 299         # youtube-dl invidious instances list
 300         r'(?:(?:www|no)\.)?invidiou\.sh',
 301         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 302         r'(?:www\.)?invidious\.kabi\.tk',
 303         r'(?:www\.)?invidious\.mastodon\.host',
 304         r'(?:www\.)?invidious\.zapashcanon\.fr',
 305         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 306         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 307         r'(?:www\.)?invidious\.himiko\.cloud',
 308         r'(?:www\.)?invidious\.reallyancient\.tech',
 309         r'(?:www\.)?invidious\.tube',
 310         r'(?:www\.)?invidiou\.site',
 311         r'(?:www\.)?invidious\.site',
 312         r'(?:www\.)?invidious\.xyz',
 313         r'(?:www\.)?invidious\.nixnet\.xyz',
 314         r'(?:www\.)?invidious\.048596\.xyz',
 315         r'(?:www\.)?invidious\.drycat\.fr',
 316         r'(?:www\.)?inv\.skyn3t\.in',
 317         r'(?:www\.)?tube\.poal\.co',
 318         r'(?:www\.)?tube\.connect\.cafe',
 319         r'(?:www\.)?vid\.wxzm\.sx',
 320         r'(?:www\.)?vid\.mint\.lgbt',
 321         r'(?:www\.)?vid\.puffyan\.us',
 322         r'(?:www\.)?yewtu\.be',
 323         r'(?:www\.)?yt\.elukerio\.org',
 324         r'(?:www\.)?yt\.lelux\.fi',
 325         r'(?:www\.)?invidious\.ggc-project\.de',
 326         r'(?:www\.)?yt\.maisputain\.ovh',
 327         r'(?:www\.)?ytprivate\.com',
 328         r'(?:www\.)?invidious\.13ad\.de',
 329         r'(?:www\.)?invidious\.toot\.koeln',
 330         r'(?:www\.)?invidious\.fdn\.fr',
 331         r'(?:www\.)?watch\.nettohikari\.com',
 332         r'(?:www\.)?invidious\.namazso\.eu',
 333         r'(?:www\.)?invidious\.silkky\.cloud',
 334         r'(?:www\.)?invidious\.exonip\.de',
 335         r'(?:www\.)?invidious\.riverside\.rocks',
 336         r'(?:www\.)?invidious\.blamefran\.net',
 337         r'(?:www\.)?invidious\.moomoo\.de',
 338         r'(?:www\.)?ytb\.trom\.tf',
 339         r'(?:www\.)?yt\.cyberhost\.uk',
 340         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 341         r'(?:www\.)?qklhadlycap4cnod\.onion',
 342         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 343         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 344         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 345         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 346         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 347         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 348         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 349         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 350         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 351         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 352         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 353         r'(?:www\.)?piped\.kavin\.rocks',
 354         r'(?:www\.)?piped\.silkky\.cloud',
 355         r'(?:www\.)?piped\.tokhmi\.xyz',
 356         r'(?:www\.)?piped\.moomoo\.me',
 357         r'(?:www\.)?il\.ax',
 358         r'(?:www\.)?piped\.syncpundit\.com',
 359         r'(?:www\.)?piped\.mha\.fi',
 360         r'(?:www\.)?piped\.mint\.lgbt',
 361         r'(?:www\.)?piped\.privacy\.com\.de',
 362     )
 363
 364     def _initialize_consent(self):
 365         cookies = self._get_cookies('https://www.youtube.com/')
 366         if cookies.get('__Secure-3PSID'):
 367             return
 368         consent_id = None
 369         consent = cookies.get('CONSENT')
 370         if consent:
 371             if 'YES' in consent.value:
 372                 return
 373             consent_id = self._search_regex(
 374                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 375         if not consent_id:
 376             consent_id = random.randint(100, 999)
 377         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 378
 379     def _initialize_pref(self):
 380         cookies = self._get_cookies('https://www.youtube.com/')
 381         pref_cookie = cookies.get('PREF')
 382         pref = {}
 383         if pref_cookie:
 384             try:
 385                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 386             except ValueError:
 387                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 388         pref.update({'hl': 'en', 'tz': 'UTC'})
 389         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 390
 391     def _real_initialize(self):
 392         self._initialize_pref()
 393         self._initialize_consent()
 394         self._check_login_required()
 395
 396     def _check_login_required(self):
 397         if self._LOGIN_REQUIRED and not self._cookies_passed:
 398             self.raise_login_required('Login details are needed to download this content', method='cookies')
 399
 400     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 401     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 402     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 403
 404     def _get_default_ytcfg(self, client='web'):
 405         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 406
 407     def _get_innertube_host(self, client='web'):
 408         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 409
 410     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 411         # try_get but with fallback to default ytcfg client values when present
 412         _func = lambda y: try_get(y, getter, expected_type)
 413         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 414
 415     def _extract_client_name(self, ytcfg, default_client='web'):
 416         return self._ytcfg_get_safe(
 417             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 418                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 419
 420     def _extract_client_version(self, ytcfg, default_client='web'):
 421         return self._ytcfg_get_safe(
 422             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 423                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 424
 425     def _extract_api_key(self, ytcfg=None, default_client='web'):
 426         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 427
 428     def _extract_context(self, ytcfg=None, default_client='web'):
 429         context = get_first(
 430             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 431         # Enforce language and tz for extraction
 432         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 433         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 434         return context
 435
 436     _SAPISID = None
 437
 438     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 439         time_now = round(time.time())
 440         if self._SAPISID is None:
 441             yt_cookies = self._get_cookies('https://www.youtube.com')
 442             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 443             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 444             sapisid_cookie = dict_get(
 445                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 446             if sapisid_cookie and sapisid_cookie.value:
 447                 self._SAPISID = sapisid_cookie.value
 448                 self.write_debug('Extracted SAPISID cookie')
 449                 # SAPISID cookie is required if not already present
 450                 if not yt_cookies.get('SAPISID'):
 451                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 452                     self._set_cookie(
 453                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 454             else:
 455                 self._SAPISID = False
 456         if not self._SAPISID:
 457             return None
 458         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 459         sapisidhash = hashlib.sha1(
 460             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 461         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 462
 463     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 464                   note='Downloading API JSON', errnote='Unable to download API page',
 465                   context=None, api_key=None, api_hostname=None, default_client='web'):
 466
 467         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 468         data.update(query)
 469         real_headers = self.generate_api_headers(default_client=default_client)
 470         real_headers.update({'content-type': 'application/json'})
 471         if headers:
 472             real_headers.update(headers)
 473         return self._download_json(
 474             f'https://{api_hostname or self._get_innertube_host(default_client)}/youtubei/v1/{ep}',
 475             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 476             data=json.dumps(data).encode('utf8'), headers=real_headers,
 477             query={'key': api_key or self._extract_api_key(), 'prettyPrint': 'false'})
 478
 479     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 480         data = self._search_regex(
 481             (fr'{self._YT_INITIAL_DATA_RE}\s*{self._YT_INITIAL_BOUNDARY_RE}',
 482              self._YT_INITIAL_DATA_RE), webpage, 'yt initial data', fatal=fatal)
 483         if data:
 484             return self._parse_json(data, item_id, fatal=fatal)
 485
 486     @staticmethod
 487     def _extract_session_index(*data):
 488         """
 489         Index of current account in account list.
 490         See: https://github.com/yt-dlp/yt-dlp/pull/519
 491         """
 492         for ytcfg in data:
 493             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 494             if session_index is not None:
 495                 return session_index
 496
 497     # Deprecated?
 498     def _extract_identity_token(self, ytcfg=None, webpage=None):
 499         if ytcfg:
 500             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 501             if token:
 502                 return token
 503         if webpage:
 504             return self._search_regex(
 505                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 506                 'identity token', default=None, fatal=False)
 507
 508     @staticmethod
 509     def _extract_account_syncid(*args):
 510         """
 511         Extract syncId required to download private playlists of secondary channels
 512         @params response and/or ytcfg
 513         """
 514         for data in args:
 515             # ytcfg includes channel_syncid if on secondary channel
 516             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 517             if delegated_sid:
 518                 return delegated_sid
 519             sync_ids = (try_get(
 520                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 521                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 522             if len(sync_ids) >= 2 and sync_ids[1]:
 523                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 524                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 525                 return sync_ids[0]
 526
 527     @staticmethod
 528     def _extract_visitor_data(*args):
 529         """
 530         Extracts visitorData from an API response or ytcfg
 531         Appears to be used to track session state
 532         """
 533         return get_first(
 534             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 535             expected_type=str)
 536
 537     @property
 538     def is_authenticated(self):
 539         return bool(self._generate_sapisidhash_header())
 540
 541     def extract_ytcfg(self, video_id, webpage):
 542         if not webpage:
 543             return {}
 544         return self._parse_json(
 545             self._search_regex(
 546                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 547                 default='{}'), video_id, fatal=False) or {}
 548
 549     def generate_api_headers(
 550             self, *, ytcfg=None, account_syncid=None, session_index=None,
 551             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 552
 553         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
 554         headers = {
 555             'X-YouTube-Client-Name': compat_str(
 556                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 557             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 558             'Origin': origin,
 559             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 560             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 561             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 562         }
 563         if session_index is None:
 564             session_index = self._extract_session_index(ytcfg)
 565         if account_syncid or session_index is not None:
 566             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 567
 568         auth = self._generate_sapisidhash_header(origin)
 569         if auth is not None:
 570             headers['Authorization'] = auth
 571             headers['X-Origin'] = origin
 572         return {h: v for h, v in headers.items() if v is not None}
 573
 574     def _download_ytcfg(self, client, video_id):
 575         url = {
 576             'web': 'https://www.youtube.com',
 577             'web_music': 'https://music.youtube.com',
 578             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 579         }.get(client)
 580         if not url:
 581             return {}
 582         webpage = self._download_webpage(
 583             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 584         return self.extract_ytcfg(video_id, webpage) or {}
 585
 586     @staticmethod
 587     def _build_api_continuation_query(continuation, ctp=None):
 588         query = {
 589             'continuation': continuation
 590         }
 591         # TODO: Inconsistency with clickTrackingParams.
 592         # Currently we have a fixed ctp contained within context (from ytcfg)
 593         # and a ctp in root query for continuation.
 594         if ctp:
 595             query['clickTracking'] = {'clickTrackingParams': ctp}
 596         return query
 597
 598     @classmethod
 599     def _extract_next_continuation_data(cls, renderer):
 600         next_continuation = try_get(
 601             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 602                        lambda x: x['continuation']['reloadContinuationData']), dict)
 603         if not next_continuation:
 604             return
 605         continuation = next_continuation.get('continuation')
 606         if not continuation:
 607             return
 608         ctp = next_continuation.get('clickTrackingParams')
 609         return cls._build_api_continuation_query(continuation, ctp)
 610
 611     @classmethod
 612     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 613         if isinstance(continuation_ep, dict):
 614             continuation = try_get(
 615                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 616             if not continuation:
 617                 return
 618             ctp = continuation_ep.get('clickTrackingParams')
 619             return cls._build_api_continuation_query(continuation, ctp)
 620
 621     @classmethod
 622     def _extract_continuation(cls, renderer):
 623         next_continuation = cls._extract_next_continuation_data(renderer)
 624         if next_continuation:
 625             return next_continuation
 626
 627         contents = []
 628         for key in ('contents', 'items'):
 629             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 630
 631         for content in contents:
 632             if not isinstance(content, dict):
 633                 continue
 634             continuation_ep = try_get(
 635                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 636                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 637                 dict)
 638             continuation = cls._extract_continuation_ep_data(continuation_ep)
 639             if continuation:
 640                 return continuation
 641
 642     @classmethod
 643     def _extract_alerts(cls, data):
 644         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 645             if not isinstance(alert_dict, dict):
 646                 continue
 647             for alert in alert_dict.values():
 648                 alert_type = alert.get('type')
 649                 if not alert_type:
 650                     continue
 651                 message = cls._get_text(alert, 'text')
 652                 if message:
 653                     yield alert_type, message
 654
 655     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 656         errors = []
 657         warnings = []
 658         for alert_type, alert_message in alerts:
 659             if alert_type.lower() == 'error' and fatal:
 660                 errors.append([alert_type, alert_message])
 661             else:
 662                 warnings.append([alert_type, alert_message])
 663
 664         for alert_type, alert_message in (warnings + errors[:-1]):
 665             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 666         if errors:
 667             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 668
 669     def _extract_and_report_alerts(self, data, *args, **kwargs):
 670         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 671
 672     def _extract_badges(self, renderer: dict):
 673         badges = set()
 674         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 675             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 676             if label:
 677                 badges.add(label.lower())
 678         return badges
 679
 680     @staticmethod
 681     def _get_text(data, *path_list, max_runs=None):
 682         for path in path_list or [None]:
 683             if path is None:
 684                 obj = [data]
 685             else:
 686                 obj = traverse_obj(data, path, default=[])
 687                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 688                     obj = [obj]
 689             for item in obj:
 690                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 691                 if text:
 692                     return text
 693                 runs = try_get(item, lambda x: x['runs'], list) or []
 694                 if not runs and isinstance(item, list):
 695                     runs = item
 696
 697                 runs = runs[:min(len(runs), max_runs or len(runs))]
 698                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 699                 if text:
 700                     return text
 701
 702     def _get_count(self, data, *path_list):
 703         count_text = self._get_text(data, *path_list) or ''
 704         count = parse_count(count_text)
 705         if count is None:
 706             count = str_to_int(
 707                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 708         return count
 709
 710     @staticmethod
 711     def _extract_thumbnails(data, *path_list):
 712         """
 713         Extract thumbnails from thumbnails dict
 714         @param path_list: path list to level that contains 'thumbnails' key
 715         """
 716         thumbnails = []
 717         for path in path_list or [()]:
 718             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 719                 thumbnail_url = url_or_none(thumbnail.get('url'))
 720                 if not thumbnail_url:
 721                     continue
 722                 # Sometimes youtube gives a wrong thumbnail URL. See:
 723                 # https://github.com/yt-dlp/yt-dlp/issues/233
 724                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 725                 if 'maxresdefault' in thumbnail_url:
 726                     thumbnail_url = thumbnail_url.split('?')[0]
 727                 thumbnails.append({
 728                     'url': thumbnail_url,
 729                     'height': int_or_none(thumbnail.get('height')),
 730                     'width': int_or_none(thumbnail.get('width')),
 731                 })
 732         return thumbnails
 733
 734     @staticmethod
 735     def extract_relative_time(relative_time_text):
 736         """
 737         Extracts a relative time from string and converts to dt object
 738         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 739         """
 740         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 741         if mobj:
 742             start = mobj.group('start')
 743             if start:
 744                 return datetime_from_str(start)
 745             try:
 746                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 747             except ValueError:
 748                 return None
 749
 750     def _extract_time_text(self, renderer, *path_list):
 751         """@returns (timestamp, time_text)"""
 752         text = self._get_text(renderer, *path_list) or ''
 753         dt = self.extract_relative_time(text)
 754         timestamp = None
 755         if isinstance(dt, datetime.datetime):
 756             timestamp = calendar.timegm(dt.timetuple())
 757
 758         if timestamp is None:
 759             timestamp = (
 760                 unified_timestamp(text) or unified_timestamp(
 761                     self._search_regex(
 762                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 763                         text.lower(), 'time text', default=None)))
 764
 765         if text and timestamp is None:
 766             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 767         return timestamp, text
 768
 769     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 770                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 771                           default_client='web'):
 772         response = None
 773         last_error = None
 774         count = -1
 775         retries = self.get_param('extractor_retries', 3)
 776         if check_get_keys is None:
 777             check_get_keys = []
 778         while count < retries:
 779             count += 1
 780             if last_error:
 781                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 782             try:
 783                 response = self._call_api(
 784                     ep=ep, fatal=True, headers=headers,
 785                     video_id=item_id, query=query,
 786                     context=self._extract_context(ytcfg, default_client),
 787                     api_key=self._extract_api_key(ytcfg, default_client),
 788                     api_hostname=api_hostname, default_client=default_client,
 789                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 790             except ExtractorError as e:
 791                 if isinstance(e.cause, network_exceptions):
 792                     if isinstance(e.cause, compat_HTTPError):
 793                         first_bytes = e.cause.read(512)
 794                         if not is_html(first_bytes):
 795                             yt_error = try_get(
 796                                 self._parse_json(
 797                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 798                                 lambda x: x['error']['message'], compat_str)
 799                             if yt_error:
 800                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 801                     # Downloading page may result in intermittent 5xx HTTP error
 802                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 803                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 804                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 805                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 806                         last_error = error_to_compat_str(e.cause or e.msg)
 807                         if count < retries:
 808                             continue
 809                 if fatal:
 810                     raise
 811                 else:
 812                     self.report_warning(error_to_compat_str(e))
 813                     return
 814
 815             else:
 816                 try:
 817                     self._extract_and_report_alerts(response, only_once=True)
 818                 except ExtractorError as e:
 819                     # YouTube servers may return errors we want to retry on in a 200 OK response
 820                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 821                     if 'unknown error' in e.msg.lower():
 822                         last_error = e.msg
 823                         continue
 824                     if fatal:
 825                         raise
 826                     self.report_warning(error_to_compat_str(e))
 827                     return
 828                 if not check_get_keys or dict_get(response, check_get_keys):
 829                     break
 830                 # Youtube sometimes sends incomplete data
 831                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 832                 last_error = 'Incomplete data received'
 833                 if count >= retries:
 834                     if fatal:
 835                         raise ExtractorError(last_error)
 836                     else:
 837                         self.report_warning(last_error)
 838                         return
 839         return response
 840
 841     @staticmethod
 842     def is_music_url(url):
 843         return re.match(r'https?://music\.youtube\.com/', url) is not None
 844
 845     def _extract_video(self, renderer):
 846         video_id = renderer.get('videoId')
 847         title = self._get_text(renderer, 'title')
 848         description = self._get_text(renderer, 'descriptionSnippet')
 849         duration = parse_duration(self._get_text(
 850             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 851         if duration is None:
 852             duration = parse_duration(self._search_regex(
 853                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 854                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 855                 video_id, default=None, group='duration'))
 856
 857         view_count = self._get_count(renderer, 'viewCountText')
 858
 859         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 860         channel_id = traverse_obj(
 861             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 862             expected_type=str, get_all=False)
 863         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 864         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 865         overlay_style = traverse_obj(
 866             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 867             get_all=False, expected_type=str)
 868         badges = self._extract_badges(renderer)
 869         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 870         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 871             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 872             expected_type=str)) or ''
 873         url = f'https://www.youtube.com/watch?v={video_id}'
 874         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 875             url = f'https://www.youtube.com/shorts/{video_id}'
 876
 877         return {
 878             '_type': 'url',
 879             'ie_key': YoutubeIE.ie_key(),
 880             'id': video_id,
 881             'url': url,
 882             'title': title,
 883             'description': description,
 884             'duration': duration,
 885             'view_count': view_count,
 886             'uploader': uploader,
 887             'channel_id': channel_id,
 888             'thumbnails': thumbnails,
 889             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 890                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 891                             else None),
 892             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 893                             else 'was_live' if 'streamed' in time_text.lower()
 894                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 895                             else None),
 896             'release_timestamp': scheduled_timestamp,
 897             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 898         }
 899
 900
 901 class YoutubeIE(YoutubeBaseInfoExtractor):
 902     IE_DESC = 'YouTube'
 903     _VALID_URL = r"""(?x)^
 904                      (
 905                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 906                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 907                             (?:www\.)?deturl\.com/www\.youtube\.com|
 908                             (?:www\.)?pwnyoutube\.com|
 909                             (?:www\.)?hooktube\.com|
 910                             (?:www\.)?yourepeat\.com|
 911                             tube\.majestyc\.net|
 912                             %(invidious)s|
 913                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 914                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 915                          (?:                                                  # the various things that can precede the ID:
 916                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 917                              |(?:                                             # or the v= param in all its forms
 918                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 919                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 920                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 921                                  v=
 922                              )
 923                          ))
 924                          |(?:
 925                             youtu\.be|                                        # just youtu.be/xxxx
 926                             vid\.plus|                                        # or vid.plus/xxxx
 927                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 928                             %(invidious)s
 929                          )/
 930                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 931                          )
 932                      )?                                                       # all until now is optional -> you can pass the naked ID
 933                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 934                      (?(1).+)?                                                # if we found the ID, everything can follow
 935                      (?:\#|$)""" % {
 936         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 937     }
 938     _PLAYER_INFO_RE = (
 939         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 940         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 941         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 942     )
 943     _formats = {
 944         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 945         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 946         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 947         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 948         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 949         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 950         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 951         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 953         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 954         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 955         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 956         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 957         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 958         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 959         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 960         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 961         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 962
 963
 964         # 3D videos
 965         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 966         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 967         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 968         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 969         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 970         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 971         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 972
 973         # Apple HTTP Live Streaming
 974         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 975         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 976         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 977         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 978         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 979         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 980         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 981         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 982
 983         # DASH mp4 video
 984         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 985         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 986         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 988         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 989         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 990         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 991         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 992         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 993         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 994         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 995         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 996
 997         # Dash mp4 audio
 998         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 999         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1000         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1001         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1002         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1003         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1004         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1005
1006         # Dash webm
1007         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1008         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1009         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1010         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1011         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1012         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1013         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1014         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1015         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1016         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1017         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1018         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1020         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1021         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1022         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1023         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1024         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1025         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1026         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1027         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1028         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1029
1030         # Dash webm audio
1031         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1032         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1033
1034         # Dash webm audio with opus inside
1035         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1036         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1037         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1038
1039         # RTMP (unnamed)
1040         '_rtmp': {'protocol': 'rtmp'},
1041
1042         # av01 video only formats sometimes served with "unknown" codecs
1043         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1044         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1045         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1046         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1047         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1048         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1049         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1050         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1051     }
1052     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1053
1054     _GEO_BYPASS = False
1055
1056     IE_NAME = 'youtube'
1057     _TESTS = [
1058         {
1059             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1060             'info_dict': {
1061                 'id': 'BaW_jenozKc',
1062                 'ext': 'mp4',
1063                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1064                 'uploader': 'Philipp Hagemeister',
1065                 'uploader_id': 'phihag',
1066                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1067                 'channel': 'Philipp Hagemeister',
1068                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1069                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1070                 'upload_date': '20121002',
1071                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1072                 'categories': ['Science & Technology'],
1073                 'tags': ['youtube-dl'],
1074                 'duration': 10,
1075                 'view_count': int,
1076                 'like_count': int,
1077                 'availability': 'public',
1078                 'playable_in_embed': True,
1079                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1080                 'live_status': 'not_live',
1081                 'age_limit': 0,
1082                 'start_time': 1,
1083                 'end_time': 9,
1084                 'channel_follower_count': int
1085             }
1086         },
1087         {
1088             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1089             'note': 'Embed-only video (#1746)',
1090             'info_dict': {
1091                 'id': 'yZIXLfi8CZQ',
1092                 'ext': 'mp4',
1093                 'upload_date': '20120608',
1094                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1095                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1096                 'uploader': 'SET India',
1097                 'uploader_id': 'setindia',
1098                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1099                 'age_limit': 18,
1100             },
1101             'skip': 'Private video',
1102         },
1103         {
1104             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1105             'note': 'Use the first video ID in the URL',
1106             'info_dict': {
1107                 'id': 'BaW_jenozKc',
1108                 'ext': 'mp4',
1109                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1110                 'uploader': 'Philipp Hagemeister',
1111                 'uploader_id': 'phihag',
1112                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1113                 'channel': 'Philipp Hagemeister',
1114                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1115                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1116                 'upload_date': '20121002',
1117                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1118                 'categories': ['Science & Technology'],
1119                 'tags': ['youtube-dl'],
1120                 'duration': 10,
1121                 'view_count': int,
1122                 'like_count': int,
1123                 'availability': 'public',
1124                 'playable_in_embed': True,
1125                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1126                 'live_status': 'not_live',
1127                 'age_limit': 0,
1128                 'channel_follower_count': int
1129             },
1130             'params': {
1131                 'skip_download': True,
1132             },
1133         },
1134         {
1135             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1136             'note': '256k DASH audio (format 141) via DASH manifest',
1137             'info_dict': {
1138                 'id': 'a9LDPn-MO4I',
1139                 'ext': 'm4a',
1140                 'upload_date': '20121002',
1141                 'uploader_id': '8KVIDEO',
1142                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1143                 'description': '',
1144                 'uploader': '8KVIDEO',
1145                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1146             },
1147             'params': {
1148                 'youtube_include_dash_manifest': True,
1149                 'format': '141',
1150             },
1151             'skip': 'format 141 not served anymore',
1152         },
1153         # DASH manifest with encrypted signature
1154         {
1155             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1156             'info_dict': {
1157                 'id': 'IB3lcPjvWLA',
1158                 'ext': 'm4a',
1159                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1160                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1161                 'duration': 244,
1162                 'uploader': 'AfrojackVEVO',
1163                 'uploader_id': 'AfrojackVEVO',
1164                 'upload_date': '20131011',
1165                 'abr': 129.495,
1166                 'like_count': int,
1167                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1168                 'playable_in_embed': True,
1169                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1170                 'view_count': int,
1171                 'track': 'The Spark',
1172                 'live_status': 'not_live',
1173                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1174                 'channel': 'Afrojack',
1175                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1176                 'tags': 'count:19',
1177                 'availability': 'public',
1178                 'categories': ['Music'],
1179                 'age_limit': 0,
1180                 'alt_title': 'The Spark',
1181                 'channel_follower_count': int
1182             },
1183             'params': {
1184                 'youtube_include_dash_manifest': True,
1185                 'format': '141/bestaudio[ext=m4a]',
1186             },
1187         },
1188         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1189         {
1190             'note': 'Embed allowed age-gate video',
1191             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1192             'info_dict': {
1193                 'id': 'HtVdAasjOgU',
1194                 'ext': 'mp4',
1195                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1196                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1197                 'duration': 142,
1198                 'uploader': 'The Witcher',
1199                 'uploader_id': 'WitcherGame',
1200                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1201                 'upload_date': '20140605',
1202                 'age_limit': 18,
1203                 'categories': ['Gaming'],
1204                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1205                 'availability': 'needs_auth',
1206                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1207                 'like_count': int,
1208                 'channel': 'The Witcher',
1209                 'live_status': 'not_live',
1210                 'tags': 'count:17',
1211                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1212                 'playable_in_embed': True,
1213                 'view_count': int,
1214                 'channel_follower_count': int
1215             },
1216         },
1217         {
1218             'note': 'Age-gate video with embed allowed in public site',
1219             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1220             'info_dict': {
1221                 'id': 'HsUATh_Nc2U',
1222                 'ext': 'mp4',
1223                 'title': 'Godzilla 2 (Official Video)',
1224                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1225                 'upload_date': '20200408',
1226                 'uploader_id': 'FlyingKitty900',
1227                 'uploader': 'FlyingKitty',
1228                 'age_limit': 18,
1229                 'availability': 'needs_auth',
1230                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1231                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1232                 'channel': 'FlyingKitty',
1233                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1234                 'view_count': int,
1235                 'categories': ['Entertainment'],
1236                 'live_status': 'not_live',
1237                 'tags': ['Flyingkitty', 'godzilla 2'],
1238                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1239                 'like_count': int,
1240                 'duration': 177,
1241                 'playable_in_embed': True,
1242                 'channel_follower_count': int
1243             },
1244         },
1245         {
1246             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1247             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1248             'info_dict': {
1249                 'id': 'Tq92D6wQ1mg',
1250                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1251                 'ext': 'mp4',
1252                 'upload_date': '20191228',
1253                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1254                 'uploader': 'Projekt Melody',
1255                 'description': 'md5:17eccca93a786d51bc67646756894066',
1256                 'age_limit': 18,
1257                 'like_count': int,
1258                 'availability': 'needs_auth',
1259                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1260                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1261                 'view_count': int,
1262                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1263                 'channel': 'Projekt Melody',
1264                 'live_status': 'not_live',
1265                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1266                 'playable_in_embed': True,
1267                 'categories': ['Entertainment'],
1268                 'duration': 106,
1269                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1270                 'channel_follower_count': int
1271             },
1272         },
1273         {
1274             'note': 'Non-Agegated non-embeddable video',
1275             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1276             'info_dict': {
1277                 'id': 'MeJVWBSsPAY',
1278                 'ext': 'mp4',
1279                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1280                 'uploader': 'Herr Lurik',
1281                 'uploader_id': 'st3in234',
1282                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1283                 'upload_date': '20130730',
1284                 'track': 'Such mich find mich',
1285                 'age_limit': 0,
1286                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1287                 'like_count': int,
1288                 'playable_in_embed': False,
1289                 'creator': 'OOMPH!',
1290                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1291                 'view_count': int,
1292                 'alt_title': 'Such mich find mich',
1293                 'duration': 210,
1294                 'channel': 'Herr Lurik',
1295                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1296                 'categories': ['Music'],
1297                 'availability': 'public',
1298                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1299                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1300                 'live_status': 'not_live',
1301                 'artist': 'OOMPH!',
1302                 'channel_follower_count': int
1303             },
1304         },
1305         {
1306             'note': 'Non-bypassable age-gated video',
1307             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1308             'only_matching': True,
1309         },
1310         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1311         # YouTube Red ad is not captured for creator
1312         {
1313             'url': '__2ABJjxzNo',
1314             'info_dict': {
1315                 'id': '__2ABJjxzNo',
1316                 'ext': 'mp4',
1317                 'duration': 266,
1318                 'upload_date': '20100430',
1319                 'uploader_id': 'deadmau5',
1320                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1321                 'creator': 'deadmau5',
1322                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1323                 'uploader': 'deadmau5',
1324                 'title': 'Deadmau5 - Some Chords (HD)',
1325                 'alt_title': 'Some Chords',
1326                 'availability': 'public',
1327                 'tags': 'count:14',
1328                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1329                 'view_count': int,
1330                 'live_status': 'not_live',
1331                 'channel': 'deadmau5',
1332                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1333                 'like_count': int,
1334                 'track': 'Some Chords',
1335                 'artist': 'deadmau5',
1336                 'playable_in_embed': True,
1337                 'age_limit': 0,
1338                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1339                 'categories': ['Music'],
1340                 'album': 'Some Chords',
1341                 'channel_follower_count': int
1342             },
1343             'expected_warnings': [
1344                 'DASH manifest missing',
1345             ]
1346         },
1347         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1348         {
1349             'url': 'lqQg6PlCWgI',
1350             'info_dict': {
1351                 'id': 'lqQg6PlCWgI',
1352                 'ext': 'mp4',
1353                 'duration': 6085,
1354                 'upload_date': '20150827',
1355                 'uploader_id': 'olympic',
1356                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1357                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1358                 'uploader': 'Olympics',
1359                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1360                 'like_count': int,
1361                 'release_timestamp': 1343767800,
1362                 'playable_in_embed': True,
1363                 'categories': ['Sports'],
1364                 'release_date': '20120731',
1365                 'channel': 'Olympics',
1366                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1367                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1368                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1369                 'age_limit': 0,
1370                 'availability': 'public',
1371                 'live_status': 'was_live',
1372                 'view_count': int,
1373                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1374                 'channel_follower_count': int
1375             },
1376             'params': {
1377                 'skip_download': 'requires avconv',
1378             }
1379         },
1380         # Non-square pixels
1381         {
1382             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1383             'info_dict': {
1384                 'id': '_b-2C3KPAM0',
1385                 'ext': 'mp4',
1386                 'stretched_ratio': 16 / 9.,
1387                 'duration': 85,
1388                 'upload_date': '20110310',
1389                 'uploader_id': 'AllenMeow',
1390                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1391                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1392                 'uploader': '孫ᄋᄅ',
1393                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1394                 'playable_in_embed': True,
1395                 'channel': '孫ᄋᄅ',
1396                 'age_limit': 0,
1397                 'tags': 'count:11',
1398                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1399                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1400                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1401                 'view_count': int,
1402                 'categories': ['People & Blogs'],
1403                 'like_count': int,
1404                 'live_status': 'not_live',
1405                 'availability': 'unlisted',
1406                 'channel_follower_count': int
1407             },
1408         },
1409         # url_encoded_fmt_stream_map is empty string
1410         {
1411             'url': 'qEJwOuvDf7I',
1412             'info_dict': {
1413                 'id': 'qEJwOuvDf7I',
1414                 'ext': 'webm',
1415                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1416                 'description': '',
1417                 'upload_date': '20150404',
1418                 'uploader_id': 'spbelect',
1419                 'uploader': 'Наблюдатели Петербурга',
1420             },
1421             'params': {
1422                 'skip_download': 'requires avconv',
1423             },
1424             'skip': 'This live event has ended.',
1425         },
1426         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1427         {
1428             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1429             'info_dict': {
1430                 'id': 'FIl7x6_3R5Y',
1431                 'ext': 'webm',
1432                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1433                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1434                 'duration': 220,
1435                 'upload_date': '20150625',
1436                 'uploader_id': 'dorappi2000',
1437                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1438                 'uploader': 'dorappi2000',
1439                 'formats': 'mincount:31',
1440             },
1441             'skip': 'not actual anymore',
1442         },
1443         # DASH manifest with segment_list
1444         {
1445             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1446             'md5': '8ce563a1d667b599d21064e982ab9e31',
1447             'info_dict': {
1448                 'id': 'CsmdDsKjzN8',
1449                 'ext': 'mp4',
1450                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1451                 'uploader': 'Airtek',
1452                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1453                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1454                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1455             },
1456             'params': {
1457                 'youtube_include_dash_manifest': True,
1458                 'format': '135',  # bestvideo
1459             },
1460             'skip': 'This live event has ended.',
1461         },
1462         {
1463             # Multifeed videos (multiple cameras), URL is for Main Camera
1464             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1465             'info_dict': {
1466                 'id': 'jvGDaLqkpTg',
1467                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1468                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1469             },
1470             'playlist': [{
1471                 'info_dict': {
1472                     'id': 'jvGDaLqkpTg',
1473                     'ext': 'mp4',
1474                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1475                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1476                     'duration': 10643,
1477                     'upload_date': '20161111',
1478                     'uploader': 'Team PGP',
1479                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1480                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1481                 },
1482             }, {
1483                 'info_dict': {
1484                     'id': '3AKt1R1aDnw',
1485                     'ext': 'mp4',
1486                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1487                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1488                     'duration': 10991,
1489                     'upload_date': '20161111',
1490                     'uploader': 'Team PGP',
1491                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1492                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1493                 },
1494             }, {
1495                 'info_dict': {
1496                     'id': 'RtAMM00gpVc',
1497                     'ext': 'mp4',
1498                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1499                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1500                     'duration': 10995,
1501                     'upload_date': '20161111',
1502                     'uploader': 'Team PGP',
1503                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1504                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1505                 },
1506             }, {
1507                 'info_dict': {
1508                     'id': '6N2fdlP3C5U',
1509                     'ext': 'mp4',
1510                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1511                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1512                     'duration': 10990,
1513                     'upload_date': '20161111',
1514                     'uploader': 'Team PGP',
1515                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1516                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1517                 },
1518             }],
1519             'params': {
1520                 'skip_download': True,
1521             },
1522             'skip': 'Not multifeed anymore',
1523         },
1524         {
1525             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1526             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1527             'info_dict': {
1528                 'id': 'gVfLd0zydlo',
1529                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1530             },
1531             'playlist_count': 2,
1532             'skip': 'Not multifeed anymore',
1533         },
1534         {
1535             'url': 'https://vid.plus/FlRa-iH7PGw',
1536             'only_matching': True,
1537         },
1538         {
1539             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1540             'only_matching': True,
1541         },
1542         {
1543             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1544             # Also tests cut-off URL expansion in video description (see
1545             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1546             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1547             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1548             'info_dict': {
1549                 'id': 'lsguqyKfVQg',
1550                 'ext': 'mp4',
1551                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1552                 'alt_title': 'Dark Walk',
1553                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1554                 'duration': 133,
1555                 'upload_date': '20151119',
1556                 'uploader_id': 'IronSoulElf',
1557                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1558                 'uploader': 'IronSoulElf',
1559                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1560                 'track': 'Dark Walk',
1561                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1562                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1563                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1564                 'categories': ['Film & Animation'],
1565                 'view_count': int,
1566                 'live_status': 'not_live',
1567                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1568                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1569                 'tags': 'count:13',
1570                 'availability': 'public',
1571                 'channel': 'IronSoulElf',
1572                 'playable_in_embed': True,
1573                 'like_count': int,
1574                 'age_limit': 0,
1575                 'channel_follower_count': int
1576             },
1577             'params': {
1578                 'skip_download': True,
1579             },
1580         },
1581         {
1582             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1583             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1584             'only_matching': True,
1585         },
1586         {
1587             # Video with yt:stretch=17:0
1588             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1589             'info_dict': {
1590                 'id': 'Q39EVAstoRM',
1591                 'ext': 'mp4',
1592                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1593                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1594                 'upload_date': '20151107',
1595                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1596                 'uploader': 'CH GAMER DROID',
1597             },
1598             'params': {
1599                 'skip_download': True,
1600             },
1601             'skip': 'This video does not exist.',
1602         },
1603         {
1604             # Video with incomplete 'yt:stretch=16:'
1605             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1606             'only_matching': True,
1607         },
1608         {
1609             # Video licensed under Creative Commons
1610             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1611             'info_dict': {
1612                 'id': 'M4gD1WSo5mA',
1613                 'ext': 'mp4',
1614                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1615                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1616                 'duration': 721,
1617                 'upload_date': '20150128',
1618                 'uploader_id': 'BerkmanCenter',
1619                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1620                 'uploader': 'The Berkman Klein Center for Internet & Society',
1621                 'license': 'Creative Commons Attribution license (reuse allowed)',
1622                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1623                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1624                 'like_count': int,
1625                 'age_limit': 0,
1626                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1627                 'channel': 'The Berkman Klein Center for Internet & Society',
1628                 'availability': 'public',
1629                 'view_count': int,
1630                 'categories': ['Education'],
1631                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1632                 'live_status': 'not_live',
1633                 'playable_in_embed': True,
1634                 'channel_follower_count': int
1635             },
1636             'params': {
1637                 'skip_download': True,
1638             },
1639         },
1640         {
1641             # Channel-like uploader_url
1642             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1643             'info_dict': {
1644                 'id': 'eQcmzGIKrzg',
1645                 'ext': 'mp4',
1646                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1647                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1648                 'duration': 4060,
1649                 'upload_date': '20151120',
1650                 'uploader': 'Bernie Sanders',
1651                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1652                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1653                 'license': 'Creative Commons Attribution license (reuse allowed)',
1654                 'playable_in_embed': True,
1655                 'tags': 'count:12',
1656                 'like_count': int,
1657                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1658                 'age_limit': 0,
1659                 'availability': 'public',
1660                 'categories': ['News & Politics'],
1661                 'channel': 'Bernie Sanders',
1662                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1663                 'view_count': int,
1664                 'live_status': 'not_live',
1665                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1666                 'channel_follower_count': int
1667             },
1668             'params': {
1669                 'skip_download': True,
1670             },
1671         },
1672         {
1673             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1674             'only_matching': True,
1675         },
1676         {
1677             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1678             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1679             'only_matching': True,
1680         },
1681         {
1682             # Rental video preview
1683             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1684             'info_dict': {
1685                 'id': 'uGpuVWrhIzE',
1686                 'ext': 'mp4',
1687                 'title': 'Piku - Trailer',
1688                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1689                 'upload_date': '20150811',
1690                 'uploader': 'FlixMatrix',
1691                 'uploader_id': 'FlixMatrixKaravan',
1692                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1693                 'license': 'Standard YouTube License',
1694             },
1695             'params': {
1696                 'skip_download': True,
1697             },
1698             'skip': 'This video is not available.',
1699         },
1700         {
1701             # YouTube Red video with episode data
1702             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1703             'info_dict': {
1704                 'id': 'iqKdEhx-dD4',
1705                 'ext': 'mp4',
1706                 'title': 'Isolation - Mind Field (Ep 1)',
1707                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1708                 'duration': 2085,
1709                 'upload_date': '20170118',
1710                 'uploader': 'Vsauce',
1711                 'uploader_id': 'Vsauce',
1712                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1713                 'series': 'Mind Field',
1714                 'season_number': 1,
1715                 'episode_number': 1,
1716                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1717                 'tags': 'count:12',
1718                 'view_count': int,
1719                 'availability': 'public',
1720                 'age_limit': 0,
1721                 'channel': 'Vsauce',
1722                 'episode': 'Episode 1',
1723                 'categories': ['Entertainment'],
1724                 'season': 'Season 1',
1725                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1726                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1727                 'like_count': int,
1728                 'playable_in_embed': True,
1729                 'live_status': 'not_live',
1730                 'channel_follower_count': int
1731             },
1732             'params': {
1733                 'skip_download': True,
1734             },
1735             'expected_warnings': [
1736                 'Skipping DASH manifest',
1737             ],
1738         },
1739         {
1740             # The following content has been identified by the YouTube community
1741             # as inappropriate or offensive to some audiences.
1742             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1743             'info_dict': {
1744                 'id': '6SJNVb0GnPI',
1745                 'ext': 'mp4',
1746                 'title': 'Race Differences in Intelligence',
1747                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1748                 'duration': 965,
1749                 'upload_date': '20140124',
1750                 'uploader': 'New Century Foundation',
1751                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1752                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1753             },
1754             'params': {
1755                 'skip_download': True,
1756             },
1757             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1758         },
1759         {
1760             # itag 212
1761             'url': '1t24XAntNCY',
1762             'only_matching': True,
1763         },
1764         {
1765             # geo restricted to JP
1766             'url': 'sJL6WA-aGkQ',
1767             'only_matching': True,
1768         },
1769         {
1770             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1771             'only_matching': True,
1772         },
1773         {
1774             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1775             'only_matching': True,
1776         },
1777         {
1778             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1779             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1780             'only_matching': True,
1781         },
1782         {
1783             # DRM protected
1784             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1785             'only_matching': True,
1786         },
1787         {
1788             # Video with unsupported adaptive stream type formats
1789             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1790             'info_dict': {
1791                 'id': 'Z4Vy8R84T1U',
1792                 'ext': 'mp4',
1793                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1794                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1795                 'duration': 433,
1796                 'upload_date': '20130923',
1797                 'uploader': 'Amelia Putri Harwita',
1798                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1799                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1800                 'formats': 'maxcount:10',
1801             },
1802             'params': {
1803                 'skip_download': True,
1804                 'youtube_include_dash_manifest': False,
1805             },
1806             'skip': 'not actual anymore',
1807         },
1808         {
1809             # Youtube Music Auto-generated description
1810             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1811             'info_dict': {
1812                 'id': 'MgNrAu2pzNs',
1813                 'ext': 'mp4',
1814                 'title': 'Voyeur Girl',
1815                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1816                 'upload_date': '20190312',
1817                 'uploader': 'Stephen - Topic',
1818                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1819                 'artist': 'Stephen',
1820                 'track': 'Voyeur Girl',
1821                 'album': 'it\'s too much love to know my dear',
1822                 'release_date': '20190313',
1823                 'release_year': 2019,
1824                 'alt_title': 'Voyeur Girl',
1825                 'view_count': int,
1826                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1827                 'playable_in_embed': True,
1828                 'like_count': int,
1829                 'categories': ['Music'],
1830                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1831                 'channel': 'Stephen',
1832                 'availability': 'public',
1833                 'creator': 'Stephen',
1834                 'duration': 169,
1835                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1836                 'age_limit': 0,
1837                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1838                 'tags': 'count:11',
1839                 'live_status': 'not_live',
1840                 'channel_follower_count': int
1841             },
1842             'params': {
1843                 'skip_download': True,
1844             },
1845         },
1846         {
1847             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1848             'only_matching': True,
1849         },
1850         {
1851             # invalid -> valid video id redirection
1852             'url': 'DJztXj2GPfl',
1853             'info_dict': {
1854                 'id': 'DJztXj2GPfk',
1855                 'ext': 'mp4',
1856                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1857                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1858                 'upload_date': '20090125',
1859                 'uploader': 'Prochorowka',
1860                 'uploader_id': 'Prochorowka',
1861                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1862                 'artist': 'Panjabi MC',
1863                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1864                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1865             },
1866             'params': {
1867                 'skip_download': True,
1868             },
1869             'skip': 'Video unavailable',
1870         },
1871         {
1872             # empty description results in an empty string
1873             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1874             'info_dict': {
1875                 'id': 'x41yOUIvK2k',
1876                 'ext': 'mp4',
1877                 'title': 'IMG 3456',
1878                 'description': '',
1879                 'upload_date': '20170613',
1880                 'uploader_id': 'ElevageOrVert',
1881                 'uploader': 'ElevageOrVert',
1882                 'view_count': int,
1883                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1884                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1885                 'like_count': int,
1886                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1887                 'tags': [],
1888                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1889                 'availability': 'public',
1890                 'age_limit': 0,
1891                 'categories': ['Pets & Animals'],
1892                 'duration': 7,
1893                 'playable_in_embed': True,
1894                 'live_status': 'not_live',
1895                 'channel': 'ElevageOrVert',
1896                 'channel_follower_count': int
1897             },
1898             'params': {
1899                 'skip_download': True,
1900             },
1901         },
1902         {
1903             # with '};' inside yt initial data (see [1])
1904             # see [2] for an example with '};' inside ytInitialPlayerResponse
1905             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1906             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1907             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1908             'info_dict': {
1909                 'id': 'CHqg6qOn4no',
1910                 'ext': 'mp4',
1911                 'title': 'Part 77   Sort a list of simple types in c#',
1912                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1913                 'upload_date': '20130831',
1914                 'uploader_id': 'kudvenkat',
1915                 'uploader': 'kudvenkat',
1916                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1917                 'like_count': int,
1918                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1919                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1920                 'live_status': 'not_live',
1921                 'categories': ['Education'],
1922                 'availability': 'public',
1923                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1924                 'tags': 'count:12',
1925                 'playable_in_embed': True,
1926                 'age_limit': 0,
1927                 'view_count': int,
1928                 'duration': 522,
1929                 'channel': 'kudvenkat',
1930                 'channel_follower_count': int
1931             },
1932             'params': {
1933                 'skip_download': True,
1934             },
1935         },
1936         {
1937             # another example of '};' in ytInitialData
1938             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1939             'only_matching': True,
1940         },
1941         {
1942             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1943             'only_matching': True,
1944         },
1945         {
1946             # https://github.com/ytdl-org/youtube-dl/pull/28094
1947             'url': 'OtqTfy26tG0',
1948             'info_dict': {
1949                 'id': 'OtqTfy26tG0',
1950                 'ext': 'mp4',
1951                 'title': 'Burn Out',
1952                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1953                 'upload_date': '20141120',
1954                 'uploader': 'The Cinematic Orchestra - Topic',
1955                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1956                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1957                 'artist': 'The Cinematic Orchestra',
1958                 'track': 'Burn Out',
1959                 'album': 'Every Day',
1960                 'like_count': int,
1961                 'live_status': 'not_live',
1962                 'alt_title': 'Burn Out',
1963                 'duration': 614,
1964                 'age_limit': 0,
1965                 'view_count': int,
1966                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1967                 'creator': 'The Cinematic Orchestra',
1968                 'channel': 'The Cinematic Orchestra',
1969                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1970                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1971                 'availability': 'public',
1972                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1973                 'categories': ['Music'],
1974                 'playable_in_embed': True,
1975                 'channel_follower_count': int
1976             },
1977             'params': {
1978                 'skip_download': True,
1979             },
1980         },
1981         {
1982             # controversial video, only works with bpctr when authenticated with cookies
1983             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1984             'only_matching': True,
1985         },
1986         {
1987             # controversial video, requires bpctr/contentCheckOk
1988             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1989             'info_dict': {
1990                 'id': 'SZJvDhaSDnc',
1991                 'ext': 'mp4',
1992                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1993                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1994                 'uploader': 'CBS Mornings',
1995                 'uploader_id': 'CBSThisMorning',
1996                 'upload_date': '20140716',
1997                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1998                 'duration': 170,
1999                 'categories': ['News & Politics'],
2000                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2001                 'view_count': int,
2002                 'channel': 'CBS Mornings',
2003                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2004                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2005                 'age_limit': 18,
2006                 'availability': 'needs_auth',
2007                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2008                 'like_count': int,
2009                 'live_status': 'not_live',
2010                 'playable_in_embed': True,
2011                 'channel_follower_count': int
2012             }
2013         },
2014         {
2015             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2016             'url': 'cBvYw8_A0vQ',
2017             'info_dict': {
2018                 'id': 'cBvYw8_A0vQ',
2019                 'ext': 'mp4',
2020                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2021                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2022                 'upload_date': '20201120',
2023                 'uploader': 'Walk around Japan',
2024                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2025                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2026                 'duration': 1456,
2027                 'categories': ['Travel & Events'],
2028                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2029                 'view_count': int,
2030                 'channel': 'Walk around Japan',
2031                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2032                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2033                 'age_limit': 0,
2034                 'availability': 'public',
2035                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2036                 'live_status': 'not_live',
2037                 'playable_in_embed': True,
2038                 'channel_follower_count': int
2039             },
2040             'params': {
2041                 'skip_download': True,
2042             },
2043         }, {
2044             # Has multiple audio streams
2045             'url': 'WaOKSUlf4TM',
2046             'only_matching': True
2047         }, {
2048             # Requires Premium: has format 141 when requested using YTM url
2049             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2050             'only_matching': True
2051         }, {
2052             # multiple subtitles with same lang_code
2053             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2054             'only_matching': True,
2055         }, {
2056             # Force use android client fallback
2057             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2058             'info_dict': {
2059                 'id': 'YOelRv7fMxY',
2060                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2061                 'ext': '3gp',
2062                 'upload_date': '20210624',
2063                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2064                 'uploader': 'colinfurze',
2065                 'uploader_id': 'colinfurze',
2066                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2067                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2068                 'duration': 596,
2069                 'categories': ['Entertainment'],
2070                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2071                 'view_count': int,
2072                 'channel': 'colinfurze',
2073                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2074                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2075                 'age_limit': 0,
2076                 'availability': 'public',
2077                 'like_count': int,
2078                 'live_status': 'not_live',
2079                 'playable_in_embed': True,
2080                 'channel_follower_count': int
2081             },
2082             'params': {
2083                 'format': '17',  # 3gp format available on android
2084                 'extractor_args': {'youtube': {'player_client': ['android']}},
2085             },
2086         },
2087         {
2088             # Skip download of additional client configs (remix client config in this case)
2089             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2090             'only_matching': True,
2091             'params': {
2092                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2093             },
2094         }, {
2095             # shorts
2096             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2097             'only_matching': True,
2098         }, {
2099             'note': 'Storyboards',
2100             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2101             'info_dict': {
2102                 'id': '5KLPxDtMqe8',
2103                 'ext': 'mhtml',
2104                 'format_id': 'sb0',
2105                 'title': 'Your Brain is Plastic',
2106                 'uploader_id': 'scishow',
2107                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2108                 'upload_date': '20140324',
2109                 'uploader': 'SciShow',
2110                 'like_count': int,
2111                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2112                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2113                 'view_count': int,
2114                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2115                 'playable_in_embed': True,
2116                 'tags': 'count:12',
2117                 'uploader_url': 'http://www.youtube.com/user/scishow',
2118                 'availability': 'public',
2119                 'channel': 'SciShow',
2120                 'live_status': 'not_live',
2121                 'duration': 248,
2122                 'categories': ['Education'],
2123                 'age_limit': 0,
2124                 'channel_follower_count': int
2125             }, 'params': {'format': 'mhtml', 'skip_download': True}
2126         }, {
2127             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2128             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2129             'info_dict': {
2130                 'id': '2NUZ8W2llS4',
2131                 'ext': 'mp4',
2132                 'title': 'The NP that test your phone performance 🙂',
2133                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2134                 'uploader': 'Leon Nguyen',
2135                 'uploader_id': 'VNSXIII',
2136                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2137                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2138                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2139                 'duration': 21,
2140                 'view_count': int,
2141                 'age_limit': 0,
2142                 'categories': ['Gaming'],
2143                 'tags': 'count:23',
2144                 'playable_in_embed': True,
2145                 'live_status': 'not_live',
2146                 'upload_date': '20220103',
2147                 'like_count': int,
2148                 'availability': 'public',
2149                 'channel': 'Leon Nguyen',
2150                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2151                 'channel_follower_count': int
2152             }
2153         }, {
2154             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2155             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2156             'info_dict': {
2157                 'id': 'mzZzzBU6lrM',
2158                 'ext': 'mp4',
2159                 'title': 'I Met GeorgeNotFound In Real Life...',
2160                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2161                 'uploader': 'Quackity',
2162                 'uploader_id': 'QuackityHQ',
2163                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2164                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2165                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2166                 'duration': 955,
2167                 'view_count': int,
2168                 'age_limit': 0,
2169                 'categories': ['Entertainment'],
2170                 'tags': 'count:26',
2171                 'playable_in_embed': True,
2172                 'live_status': 'not_live',
2173                 'release_timestamp': 1641172509,
2174                 'release_date': '20220103',
2175                 'upload_date': '20220103',
2176                 'like_count': int,
2177                 'availability': 'public',
2178                 'channel': 'Quackity',
2179                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2180                 'channel_follower_count': int
2181             }
2182         },
2183         {   # continuous livestream. Microformat upload date should be preferred.
2184             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2185             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2186             'info_dict': {
2187                 'id': 'kgx4WGK0oNU',
2188                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2189                 'ext': 'mp4',
2190                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2191                 'availability': 'public',
2192                 'age_limit': 0,
2193                 'release_timestamp': 1637975704,
2194                 'upload_date': '20210619',
2195                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2196                 'live_status': 'is_live',
2197                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2198                 'uploader': '阿鲍Abao',
2199                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2200                 'channel': 'Abao in Tokyo',
2201                 'channel_follower_count': int,
2202                 'release_date': '20211127',
2203                 'tags': 'count:39',
2204                 'categories': ['People & Blogs'],
2205                 'like_count': int,
2206                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2207                 'view_count': int,
2208                 'playable_in_embed': True,
2209                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2210             },
2211             'params': {'skip_download': True}
2212         }, {
2213             # Story. Requires specific player params to work.
2214             # Note: stories get removed after some period of time
2215             'url': 'https://www.youtube.com/watch?v=yN3x1t3sieA',
2216             'info_dict': {
2217                 'id': 'yN3x1t3sieA',
2218                 'ext': 'mp4',
2219                 'uploader': 'Linus Tech Tips',
2220                 'duration': 13,
2221                 'channel': 'Linus Tech Tips',
2222                 'playable_in_embed': True,
2223                 'tags': [],
2224                 'age_limit': 0,
2225                 'uploader_url': 'http://www.youtube.com/user/LinusTechTips',
2226                 'upload_date': '20220402',
2227                 'thumbnail': 'https://i.ytimg.com/vi_webp/yN3x1t3sieA/maxresdefault.webp',
2228                 'title': 'Story',
2229                 'live_status': 'not_live',
2230                 'uploader_id': 'LinusTechTips',
2231                 'view_count': int,
2232                 'description': '',
2233                 'channel_id': 'UCXuqSBlHAE6Xw-yeJA0Tunw',
2234                 'categories': ['Science & Technology'],
2235                 'channel_url': 'https://www.youtube.com/channel/UCXuqSBlHAE6Xw-yeJA0Tunw',
2236                 'availability': 'unlisted',
2237             }
2238         }
2239     ]
2240
2241     @classmethod
2242     def suitable(cls, url):
2243         from ..utils import parse_qs
2244
2245         qs = parse_qs(url)
2246         if qs.get('list', [None])[0]:
2247             return False
2248         return super().suitable(url)
2249
2250     def __init__(self, *args, **kwargs):
2251         super().__init__(*args, **kwargs)
2252         self._code_cache = {}
2253         self._player_cache = {}
2254
2255     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2256         lock = threading.Lock()
2257
2258         is_live = True
2259         start_time = time.time()
2260         formats = [f for f in formats if f.get('is_from_start')]
2261
2262         def refetch_manifest(format_id, delay):
2263             nonlocal formats, start_time, is_live
2264             if time.time() <= start_time + delay:
2265                 return
2266
2267             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2268             video_details = traverse_obj(
2269                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2270             microformats = traverse_obj(
2271                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2272                 expected_type=dict, default=[])
2273             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2274             start_time = time.time()
2275
2276         def mpd_feed(format_id, delay):
2277             """
2278             @returns (manifest_url, manifest_stream_number, is_live) or None
2279             """
2280             with lock:
2281                 refetch_manifest(format_id, delay)
2282
2283             f = next((f for f in formats if f['format_id'] == format_id), None)
2284             if not f:
2285                 if not is_live:
2286                     self.to_screen(f'{video_id}: Video is no longer live')
2287                 else:
2288                     self.report_warning(
2289                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2290                 return None
2291             return f['manifest_url'], f['manifest_stream_number'], is_live
2292
2293         for f in formats:
2294             f['is_live'] = True
2295             f['protocol'] = 'http_dash_segments_generator'
2296             f['fragments'] = functools.partial(
2297                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2298
2299     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2300         FETCH_SPAN, MAX_DURATION = 5, 432000
2301
2302         mpd_url, stream_number, is_live = None, None, True
2303
2304         begin_index = 0
2305         download_start_time = ctx.get('start') or time.time()
2306
2307         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2308         if lack_early_segments:
2309             self.report_warning(bug_reports_message(
2310                 'Starting download from the last 120 hours of the live stream since '
2311                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2312             lack_early_segments = True
2313
2314         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2315         fragments, fragment_base_url = None, None
2316
2317         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2318             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2319             # Obtain from MPD's maximum seq value
2320             old_mpd_url = mpd_url
2321             last_error = ctx.pop('last_error', None)
2322             expire_fast = immediate or last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2323             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2324                                                or (mpd_url, stream_number, False))
2325             if not refresh_sequence:
2326                 if expire_fast and not is_live:
2327                     return False, last_seq
2328                 elif old_mpd_url == mpd_url:
2329                     return True, last_seq
2330             try:
2331                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2332                     mpd_url, None, note=False, errnote=False, fatal=False)
2333             except ExtractorError:
2334                 fmts = None
2335             if not fmts:
2336                 no_fragment_score += 2
2337                 return False, last_seq
2338             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2339             fragments = fmt_info['fragments']
2340             fragment_base_url = fmt_info['fragment_base_url']
2341             assert fragment_base_url
2342
2343             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2344             return True, _last_seq
2345
2346         while is_live:
2347             fetch_time = time.time()
2348             if no_fragment_score > 30:
2349                 return
2350             if last_segment_url:
2351                 # Obtain from "X-Head-Seqnum" header value from each segment
2352                 try:
2353                     urlh = self._request_webpage(
2354                         last_segment_url, None, note=False, errnote=False, fatal=False)
2355                 except ExtractorError:
2356                     urlh = None
2357                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2358                 if last_seq is None:
2359                     no_fragment_score += 2
2360                     last_segment_url = None
2361                     continue
2362             else:
2363                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2364                 no_fragment_score += 2
2365                 if not should_continue:
2366                     continue
2367
2368             if known_idx > last_seq:
2369                 last_segment_url = None
2370                 continue
2371
2372             last_seq += 1
2373
2374             if begin_index < 0 and known_idx < 0:
2375                 # skip from the start when it's negative value
2376                 known_idx = last_seq + begin_index
2377             if lack_early_segments:
2378                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2379             try:
2380                 for idx in range(known_idx, last_seq):
2381                     # do not update sequence here or you'll get skipped some part of it
2382                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2383                     if not should_continue:
2384                         known_idx = idx - 1
2385                         raise ExtractorError('breaking out of outer loop')
2386                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2387                     yield {
2388                         'url': last_segment_url,
2389                     }
2390                 if known_idx == last_seq:
2391                     no_fragment_score += 5
2392                 else:
2393                     no_fragment_score = 0
2394                 known_idx = last_seq
2395             except ExtractorError:
2396                 continue
2397
2398             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2399
2400     def _extract_player_url(self, *ytcfgs, webpage=None):
2401         player_url = traverse_obj(
2402             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2403             get_all=False, expected_type=compat_str)
2404         if not player_url:
2405             return
2406         return urljoin('https://www.youtube.com', player_url)
2407
2408     def _download_player_url(self, video_id, fatal=False):
2409         res = self._download_webpage(
2410             'https://www.youtube.com/iframe_api',
2411             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2412         if res:
2413             player_version = self._search_regex(
2414                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2415             if player_version:
2416                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2417
2418     def _signature_cache_id(self, example_sig):
2419         """ Return a string representation of a signature """
2420         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2421
2422     @classmethod
2423     def _extract_player_info(cls, player_url):
2424         for player_re in cls._PLAYER_INFO_RE:
2425             id_m = re.search(player_re, player_url)
2426             if id_m:
2427                 break
2428         else:
2429             raise ExtractorError('Cannot identify player %r' % player_url)
2430         return id_m.group('id')
2431
2432     def _load_player(self, video_id, player_url, fatal=True):
2433         player_id = self._extract_player_info(player_url)
2434         if player_id not in self._code_cache:
2435             code = self._download_webpage(
2436                 player_url, video_id, fatal=fatal,
2437                 note='Downloading player ' + player_id,
2438                 errnote='Download of %s failed' % player_url)
2439             if code:
2440                 self._code_cache[player_id] = code
2441         return self._code_cache.get(player_id)
2442
2443     def _extract_signature_function(self, video_id, player_url, example_sig):
2444         player_id = self._extract_player_info(player_url)
2445
2446         # Read from filesystem cache
2447         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2448         assert os.path.basename(func_id) == func_id
2449
2450         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
2451         if cache_spec is not None:
2452             return lambda s: ''.join(s[i] for i in cache_spec)
2453
2454         code = self._load_player(video_id, player_url)
2455         if code:
2456             res = self._parse_sig_js(code)
2457
2458             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2459             cache_res = res(test_string)
2460             cache_spec = [ord(c) for c in cache_res]
2461
2462             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
2463             return res
2464
2465     def _print_sig_code(self, func, example_sig):
2466         if not self.get_param('youtube_print_sig_code'):
2467             return
2468
2469         def gen_sig_code(idxs):
2470             def _genslice(start, end, step):
2471                 starts = '' if start == 0 else str(start)
2472                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2473                 steps = '' if step == 1 else (':%d' % step)
2474                 return f's[{starts}{ends}{steps}]'
2475
2476             step = None
2477             # Quelch pyflakes warnings - start will be set when step is set
2478             start = '(Never used)'
2479             for i, prev in zip(idxs[1:], idxs[:-1]):
2480                 if step is not None:
2481                     if i - prev == step:
2482                         continue
2483                     yield _genslice(start, prev, step)
2484                     step = None
2485                     continue
2486                 if i - prev in [-1, 1]:
2487                     step = i - prev
2488                     start = prev
2489                     continue
2490                 else:
2491                     yield 's[%d]' % prev
2492             if step is None:
2493                 yield 's[%d]' % i
2494             else:
2495                 yield _genslice(start, i, step)
2496
2497         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2498         cache_res = func(test_string)
2499         cache_spec = [ord(c) for c in cache_res]
2500         expr_code = ' + '.join(gen_sig_code(cache_spec))
2501         signature_id_tuple = '(%s)' % (
2502             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2503         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2504                 '    return %s\n') % (signature_id_tuple, expr_code)
2505         self.to_screen('Extracted signature function:\n' + code)
2506
2507     def _parse_sig_js(self, jscode):
2508         funcname = self._search_regex(
2509             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2510              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2511              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2512              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2513              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2514              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2515              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2516              # Obsolete patterns
2517              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2518              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2519              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2520              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2521              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2522              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2523              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2524              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2525             jscode, 'Initial JS player signature function name', group='sig')
2526
2527         jsi = JSInterpreter(jscode)
2528         initial_function = jsi.extract_function(funcname)
2529         return lambda s: initial_function([s])
2530
2531     def _decrypt_signature(self, s, video_id, player_url):
2532         """Turn the encrypted s field into a working signature"""
2533
2534         if player_url is None:
2535             raise ExtractorError('Cannot decrypt signature without player_url')
2536
2537         try:
2538             player_id = (player_url, self._signature_cache_id(s))
2539             if player_id not in self._player_cache:
2540                 func = self._extract_signature_function(
2541                     video_id, player_url, s
2542                 )
2543                 self._player_cache[player_id] = func
2544             func = self._player_cache[player_id]
2545             self._print_sig_code(func, s)
2546             return func(s)
2547         except Exception as e:
2548             raise ExtractorError('Signature extraction failed: ' + traceback.format_exc(), cause=e)
2549
2550     def _decrypt_nsig(self, s, video_id, player_url):
2551         """Turn the encrypted n field into a working signature"""
2552         if player_url is None:
2553             raise ExtractorError('Cannot decrypt nsig without player_url')
2554         player_url = urljoin('https://www.youtube.com', player_url)
2555
2556         sig_id = ('nsig_value', s)
2557         if sig_id in self._player_cache:
2558             return self._player_cache[sig_id]
2559
2560         try:
2561             player_id = ('nsig', player_url)
2562             if player_id not in self._player_cache:
2563                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2564             func = self._player_cache[player_id]
2565             self._player_cache[sig_id] = func(s)
2566             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2567             return self._player_cache[sig_id]
2568         except Exception as e:
2569             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2570
2571     def _extract_n_function_name(self, jscode):
2572         nfunc, idx = self._search_regex(
2573             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2574             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2575         if not idx:
2576             return nfunc
2577         return json.loads(js_to_json(self._search_regex(
2578             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2579             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2580
2581     def _extract_n_function(self, video_id, player_url):
2582         player_id = self._extract_player_info(player_url)
2583         func_code = self._downloader.cache.load('youtube-nsig', player_id)
2584
2585         if func_code:
2586             jsi = JSInterpreter(func_code)
2587         else:
2588             jscode = self._load_player(video_id, player_url)
2589             funcname = self._extract_n_function_name(jscode)
2590             jsi = JSInterpreter(jscode)
2591             func_code = jsi.extract_function_code(funcname)
2592             self._downloader.cache.store('youtube-nsig', player_id, func_code)
2593
2594         if self.get_param('youtube_print_sig_code'):
2595             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2596
2597         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2598
2599     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2600         """
2601         Extract signatureTimestamp (sts)
2602         Required to tell API what sig/player version is in use.
2603         """
2604         sts = None
2605         if isinstance(ytcfg, dict):
2606             sts = int_or_none(ytcfg.get('STS'))
2607
2608         if not sts:
2609             # Attempt to extract from player
2610             if player_url is None:
2611                 error_msg = 'Cannot extract signature timestamp without player_url.'
2612                 if fatal:
2613                     raise ExtractorError(error_msg)
2614                 self.report_warning(error_msg)
2615                 return
2616             code = self._load_player(video_id, player_url, fatal=fatal)
2617             if code:
2618                 sts = int_or_none(self._search_regex(
2619                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2620                     'JS player signature timestamp', group='sts', fatal=fatal))
2621         return sts
2622
2623     def _mark_watched(self, video_id, player_responses):
2624         playback_url = get_first(
2625             player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2626             expected_type=url_or_none)
2627         if not playback_url:
2628             self.report_warning('Unable to mark watched')
2629             return
2630         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2631         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2632
2633         # cpn generation algorithm is reverse engineered from base.js.
2634         # In fact it works even with dummy cpn.
2635         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2636         cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2637
2638         qs.update({
2639             'ver': ['2'],
2640             'cpn': [cpn],
2641         })
2642         playback_url = compat_urlparse.urlunparse(
2643             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2644
2645         self._download_webpage(
2646             playback_url, video_id, 'Marking watched',
2647             'Unable to mark watched', fatal=False)
2648
2649     @staticmethod
2650     def _extract_urls(webpage):
2651         # Embedded YouTube player
2652         entries = [
2653             unescapeHTML(mobj.group('url'))
2654             for mobj in re.finditer(r'''(?x)
2655             (?:
2656                 <iframe[^>]+?src=|
2657                 data-video-url=|
2658                 <embed[^>]+?src=|
2659                 embedSWF\(?:\s*|
2660                 <object[^>]+data=|
2661                 new\s+SWFObject\(
2662             )
2663             (["\'])
2664                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2665                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2666             \1''', webpage)]
2667
2668         # lazyYT YouTube embed
2669         entries.extend(list(map(
2670             unescapeHTML,
2671             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2672
2673         # Wordpress "YouTube Video Importer" plugin
2674         matches = re.findall(r'''(?x)<div[^>]+
2675             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2676             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2677         entries.extend(m[-1] for m in matches)
2678
2679         return entries
2680
2681     @staticmethod
2682     def _extract_url(webpage):
2683         urls = YoutubeIE._extract_urls(webpage)
2684         return urls[0] if urls else None
2685
2686     @classmethod
2687     def extract_id(cls, url):
2688         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2689         if mobj is None:
2690             raise ExtractorError('Invalid URL: %s' % url)
2691         return mobj.group('id')
2692
2693     def _extract_chapters_from_json(self, data, duration):
2694         chapter_list = traverse_obj(
2695             data, (
2696                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2697                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2698             ), expected_type=list)
2699
2700         return self._extract_chapters(
2701             chapter_list,
2702             chapter_time=lambda chapter: float_or_none(
2703                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2704             chapter_title=lambda chapter: traverse_obj(
2705                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2706             duration=duration)
2707
2708     def _extract_chapters_from_engagement_panel(self, data, duration):
2709         content_list = traverse_obj(
2710             data,
2711             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2712             expected_type=list, default=[])
2713         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2714         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2715
2716         return next((
2717             filter(None, (
2718                 self._extract_chapters(
2719                     traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2720                     chapter_time, chapter_title, duration)
2721                 for contents in content_list
2722             ))), [])
2723
2724     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration):
2725         chapters = []
2726         last_chapter = {'start_time': 0}
2727         for idx, chapter in enumerate(chapter_list or []):
2728             title = chapter_title(chapter)
2729             start_time = chapter_time(chapter)
2730             if start_time is None:
2731                 continue
2732             last_chapter['end_time'] = start_time
2733             if start_time < last_chapter['start_time']:
2734                 if idx == 1:
2735                     chapters.pop()
2736                     self.report_warning('Invalid start time for chapter "%s"' % last_chapter['title'])
2737                 else:
2738                     self.report_warning(f'Invalid start time for chapter "{title}"')
2739                     continue
2740             last_chapter = {'start_time': start_time, 'title': title}
2741             chapters.append(last_chapter)
2742         last_chapter['end_time'] = duration
2743         return chapters
2744
2745     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
2746         return self._parse_json(self._search_regex(
2747             (fr'{regex}\s*{self._YT_INITIAL_BOUNDARY_RE}',
2748              regex), webpage, name, default='{}'), video_id, fatal=False)
2749
2750     def _extract_comment(self, comment_renderer, parent=None):
2751         comment_id = comment_renderer.get('commentId')
2752         if not comment_id:
2753             return
2754
2755         text = self._get_text(comment_renderer, 'contentText')
2756
2757         # note: timestamp is an estimate calculated from the current time and time_text
2758         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2759         author = self._get_text(comment_renderer, 'authorText')
2760         author_id = try_get(comment_renderer,
2761                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2762
2763         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2764                                                        lambda x: x['likeCount']), compat_str)) or 0
2765         author_thumbnail = try_get(comment_renderer,
2766                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2767
2768         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2769         is_favorited = 'creatorHeart' in (try_get(
2770             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2771         return {
2772             'id': comment_id,
2773             'text': text,
2774             'timestamp': timestamp,
2775             'time_text': time_text,
2776             'like_count': votes,
2777             'is_favorited': is_favorited,
2778             'author': author,
2779             'author_id': author_id,
2780             'author_thumbnail': author_thumbnail,
2781             'author_is_uploader': author_is_uploader,
2782             'parent': parent or 'root'
2783         }
2784
2785     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2786
2787         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2788
2789         def extract_header(contents):
2790             _continuation = None
2791             for content in contents:
2792                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2793                 expected_comment_count = self._get_count(
2794                     comments_header_renderer, 'countText', 'commentsCount')
2795
2796                 if expected_comment_count:
2797                     tracker['est_total'] = expected_comment_count
2798                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2799                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2800
2801                 sort_menu_item = try_get(
2802                     comments_header_renderer,
2803                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2804                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2805
2806                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2807                 if not _continuation:
2808                     continue
2809
2810                 sort_text = str_or_none(sort_menu_item.get('title'))
2811                 if not sort_text:
2812                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2813                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2814                 break
2815             return _continuation
2816
2817         def extract_thread(contents):
2818             if not parent:
2819                 tracker['current_page_thread'] = 0
2820             for content in contents:
2821                 if not parent and tracker['total_parent_comments'] >= max_parents:
2822                     yield
2823                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2824                 comment_renderer = get_first(
2825                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2826                     expected_type=dict, default={})
2827
2828                 comment = self._extract_comment(comment_renderer, parent)
2829                 if not comment:
2830                     continue
2831
2832                 tracker['running_total'] += 1
2833                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2834                 yield comment
2835
2836                 # Attempt to get the replies
2837                 comment_replies_renderer = try_get(
2838                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2839
2840                 if comment_replies_renderer:
2841                     tracker['current_page_thread'] += 1
2842                     comment_entries_iter = self._comment_entries(
2843                         comment_replies_renderer, ytcfg, video_id,
2844                         parent=comment.get('id'), tracker=tracker)
2845                     yield from itertools.islice(comment_entries_iter, min(
2846                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2847
2848         # Keeps track of counts across recursive calls
2849         if not tracker:
2850             tracker = dict(
2851                 running_total=0,
2852                 est_total=0,
2853                 current_page_thread=0,
2854                 total_parent_comments=0,
2855                 total_reply_comments=0)
2856
2857         # TODO: Deprecated
2858         # YouTube comments have a max depth of 2
2859         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2860         if max_depth:
2861             self._downloader.deprecation_warning(
2862                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2863         if max_depth == 1 and parent:
2864             return
2865
2866         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2867             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2868
2869         continuation = self._extract_continuation(root_continuation_data)
2870
2871         response = None
2872         is_forced_continuation = False
2873         is_first_continuation = parent is None
2874         if is_first_continuation and not continuation:
2875             # Sometimes you can get comments by generating the continuation yourself,
2876             # even if YouTube initially reports them being disabled - e.g. stories comments.
2877             # Note: if the comment section is actually disabled, YouTube may return a response with
2878             # required check_get_keys missing. So we will disable that check initially in this case.
2879             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2880             is_forced_continuation = True
2881
2882         for page_num in itertools.count(0):
2883             if not continuation:
2884                 break
2885             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2886             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2887             if page_num == 0:
2888                 if is_first_continuation:
2889                     note_prefix = 'Downloading comment section API JSON'
2890                 else:
2891                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2892                         tracker['current_page_thread'], comment_prog_str)
2893             else:
2894                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2895                     '       ' if parent else '', ' replies' if parent else '',
2896                     page_num, comment_prog_str)
2897
2898             response = self._extract_response(
2899                 item_id=None, query=continuation,
2900                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2901                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2902             is_forced_continuation = False
2903             continuation_contents = traverse_obj(
2904                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2905
2906             continuation = None
2907             for continuation_section in continuation_contents:
2908                 continuation_items = traverse_obj(
2909                     continuation_section,
2910                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2911                     get_all=False, expected_type=list) or []
2912                 if is_first_continuation:
2913                     continuation = extract_header(continuation_items)
2914                     is_first_continuation = False
2915                     if continuation:
2916                         break
2917                     continue
2918
2919                 for entry in extract_thread(continuation_items):
2920                     if not entry:
2921                         return
2922                     yield entry
2923                 continuation = self._extract_continuation({'contents': continuation_items})
2924                 if continuation:
2925                     break
2926
2927         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2928         if message and not parent and tracker['running_total'] == 0:
2929             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2930
2931     @staticmethod
2932     def _generate_comment_continuation(video_id):
2933         """
2934         Generates initial comment section continuation token from given video id
2935         """
2936         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2937         return base64.b64encode(token.encode()).decode()
2938
2939     def _get_comments(self, ytcfg, video_id, contents, webpage):
2940         """Entry for comment extraction"""
2941         def _real_comment_extract(contents):
2942             renderer = next((
2943                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2944                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2945             yield from self._comment_entries(renderer, ytcfg, video_id)
2946
2947         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2948         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2949
2950     @staticmethod
2951     def _get_checkok_params():
2952         return {'contentCheckOk': True, 'racyCheckOk': True}
2953
2954     @classmethod
2955     def _generate_player_context(cls, sts=None):
2956         context = {
2957             'html5Preference': 'HTML5_PREF_WANTS',
2958         }
2959         if sts is not None:
2960             context['signatureTimestamp'] = sts
2961         return {
2962             'playbackContext': {
2963                 'contentPlaybackContext': context
2964             },
2965             **cls._get_checkok_params()
2966         }
2967
2968     @staticmethod
2969     def _is_agegated(player_response):
2970         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2971             return True
2972
2973         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2974         AGE_GATE_REASONS = (
2975             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2976             'age_verification_required', 'age_check_required',  # status
2977         )
2978         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
2979
2980     @staticmethod
2981     def _is_unplayable(player_response):
2982         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
2983
2984     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
2985
2986         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
2987         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
2988         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
2989         headers = self.generate_api_headers(
2990             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
2991
2992         yt_query = {
2993             'videoId': video_id,
2994             'params': '8AEB'  # enable stories
2995         }
2996         yt_query.update(self._generate_player_context(sts))
2997         return self._extract_response(
2998             item_id=video_id, ep='player', query=yt_query,
2999             ytcfg=player_ytcfg, headers=headers, fatal=True,
3000             default_client=client,
3001             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3002         ) or None
3003
3004     def _get_requested_clients(self, url, smuggled_data):
3005         requested_clients = []
3006         default = ['android', 'web']
3007         allowed_clients = sorted(
3008             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3009             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3010         for client in self._configuration_arg('player_client'):
3011             if client in allowed_clients:
3012                 requested_clients.append(client)
3013             elif client == 'default':
3014                 requested_clients.extend(default)
3015             elif client == 'all':
3016                 requested_clients.extend(allowed_clients)
3017             else:
3018                 self.report_warning(f'Skipping unsupported client {client}')
3019         if not requested_clients:
3020             requested_clients = default
3021
3022         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3023             requested_clients.extend(
3024                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3025
3026         return orderedSet(requested_clients)
3027
3028     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3029         initial_pr = None
3030         if webpage:
3031             initial_pr = self._extract_yt_initial_variable(
3032                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
3033                 video_id, 'initial player response')
3034
3035         all_clients = set(clients)
3036         clients = clients[::-1]
3037         prs = []
3038
3039         def append_client(*client_names):
3040             """ Append the first client name that exists but not already used """
3041             for client_name in client_names:
3042                 actual_client = _split_innertube_client(client_name)[0]
3043                 if actual_client in INNERTUBE_CLIENTS:
3044                     if actual_client not in all_clients:
3045                         clients.append(client_name)
3046                         all_clients.add(actual_client)
3047                         return
3048
3049         # Android player_response does not have microFormats which are needed for
3050         # extraction of some data. So we return the initial_pr with formats
3051         # stripped out even if not requested by the user
3052         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3053         if initial_pr:
3054             pr = dict(initial_pr)
3055             pr['streamingData'] = None
3056             prs.append(pr)
3057
3058         last_error = None
3059         tried_iframe_fallback = False
3060         player_url = None
3061         while clients:
3062             client, base_client, variant = _split_innertube_client(clients.pop())
3063             player_ytcfg = master_ytcfg if client == 'web' else {}
3064             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3065                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3066
3067             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3068             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3069             if 'js' in self._configuration_arg('player_skip'):
3070                 require_js_player = False
3071                 player_url = None
3072
3073             if not player_url and not tried_iframe_fallback and require_js_player:
3074                 player_url = self._download_player_url(video_id)
3075                 tried_iframe_fallback = True
3076
3077             try:
3078                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3079                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3080             except ExtractorError as e:
3081                 if last_error:
3082                     self.report_warning(last_error)
3083                 last_error = e
3084                 continue
3085
3086             if pr:
3087                 prs.append(pr)
3088
3089             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3090             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3091                 append_client(f'{base_client}_creator')
3092             elif self._is_agegated(pr):
3093                 if variant == 'tv_embedded':
3094                     append_client(f'{base_client}_embedded')
3095                 elif not variant:
3096                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3097
3098         if last_error:
3099             if not len(prs):
3100                 raise last_error
3101             self.report_warning(last_error)
3102         return prs, player_url
3103
3104     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3105         itags, stream_ids = {}, []
3106         itag_qualities, res_qualities = {}, {}
3107         q = qualities([
3108             # Normally tiny is the smallest video-only formats. But
3109             # audio-only formats with unknown quality may get tagged as tiny
3110             'tiny',
3111             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3112             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3113         ])
3114         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3115
3116         for fmt in streaming_formats:
3117             if fmt.get('targetDurationSec'):
3118                 continue
3119
3120             itag = str_or_none(fmt.get('itag'))
3121             audio_track = fmt.get('audioTrack') or {}
3122             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3123             if stream_id in stream_ids:
3124                 continue
3125
3126             quality = fmt.get('quality')
3127             height = int_or_none(fmt.get('height'))
3128             if quality == 'tiny' or not quality:
3129                 quality = fmt.get('audioQuality', '').lower() or quality
3130             # The 3gp format (17) in android client has a quality of "small",
3131             # but is actually worse than other formats
3132             if itag == '17':
3133                 quality = 'tiny'
3134             if quality:
3135                 if itag:
3136                     itag_qualities[itag] = quality
3137                 if height:
3138                     res_qualities[height] = quality
3139             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3140             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3141             # number of fragment that would subsequently requested with (`&sq=N`)
3142             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3143                 continue
3144
3145             fmt_url = fmt.get('url')
3146             if not fmt_url:
3147                 sc = compat_parse_qs(fmt.get('signatureCipher'))
3148                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3149                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3150                 if not (sc and fmt_url and encrypted_sig):
3151                     continue
3152                 if not player_url:
3153                     continue
3154                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
3155                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
3156                 fmt_url += '&' + sp + '=' + signature
3157
3158             query = parse_qs(fmt_url)
3159             throttled = False
3160             if query.get('n'):
3161                 try:
3162                     fmt_url = update_url_query(fmt_url, {
3163                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3164                 except ExtractorError as e:
3165                     self.report_warning(
3166                         'nsig extraction failed: You may experience throttling for some formats\n'
3167                         f'n = {query["n"][0]} ; player = {player_url}\n{e}', only_once=True)
3168                     throttled = True
3169
3170             if itag:
3171                 itags[itag] = 'https'
3172                 stream_ids.append(stream_id)
3173
3174             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3175             language_preference = (
3176                 10 if audio_track.get('audioIsDefault') and 10
3177                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3178                 else -1)
3179             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3180             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3181             # Make sure to avoid false positives with small duration differences.
3182             # Eg: __2ABJjxzNo, ySuUZEjARPY
3183             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3184             if is_damaged:
3185                 self.report_warning(
3186                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3187             dct = {
3188                 'asr': int_or_none(fmt.get('audioSampleRate')),
3189                 'filesize': int_or_none(fmt.get('contentLength')),
3190                 'format_id': itag,
3191                 'format_note': join_nonempty(
3192                     '%s%s' % (audio_track.get('displayName') or '',
3193                               ' (default)' if language_preference > 0 else ''),
3194                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3195                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3196                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3197                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3198                 'fps': int_or_none(fmt.get('fps')) or None,
3199                 'height': height,
3200                 'quality': q(quality),
3201                 'has_drm': bool(fmt.get('drmFamilies')),
3202                 'tbr': tbr,
3203                 'url': fmt_url,
3204                 'width': int_or_none(fmt.get('width')),
3205                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3206                                           'desc' if language_preference < -1 else ''),
3207                 'language_preference': language_preference,
3208                 # Strictly de-prioritize damaged and 3gp formats
3209                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3210             }
3211             mime_mobj = re.match(
3212                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3213             if mime_mobj:
3214                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3215                 dct.update(parse_codecs(mime_mobj.group(2)))
3216             no_audio = dct.get('acodec') == 'none'
3217             no_video = dct.get('vcodec') == 'none'
3218             if no_audio:
3219                 dct['vbr'] = tbr
3220             if no_video:
3221                 dct['abr'] = tbr
3222             if no_audio or no_video:
3223                 dct['downloader_options'] = {
3224                     # Youtube throttles chunks >~10M
3225                     'http_chunk_size': 10485760,
3226                 }
3227                 if dct.get('ext'):
3228                     dct['container'] = dct['ext'] + '_dash'
3229             yield dct
3230
3231         live_from_start = is_live and self.get_param('live_from_start')
3232         skip_manifests = self._configuration_arg('skip')
3233         if not self.get_param('youtube_include_hls_manifest', True):
3234             skip_manifests.append('hls')
3235         if not self.get_param('youtube_include_dash_manifest', True):
3236             skip_manifests.append('dash')
3237         get_dash = 'dash' not in skip_manifests and (
3238             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3239         get_hls = not live_from_start and 'hls' not in skip_manifests
3240
3241         def process_manifest_format(f, proto, itag):
3242             if itag in itags:
3243                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3244                     return False
3245                 itag = f'{itag}-{proto}'
3246             if itag:
3247                 f['format_id'] = itag
3248                 itags[itag] = proto
3249
3250             f['quality'] = next((
3251                 q(qdict[val])
3252                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3253                 if val in qdict), -1)
3254             return True
3255
3256         for sd in streaming_data:
3257             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3258             if hls_manifest_url:
3259                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3260                     if process_manifest_format(f, 'hls', self._search_regex(
3261                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3262                         yield f
3263
3264             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3265             if dash_manifest_url:
3266                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3267                     if process_manifest_format(f, 'dash', f['format_id']):
3268                         f['filesize'] = int_or_none(self._search_regex(
3269                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3270                         if live_from_start:
3271                             f['is_from_start'] = True
3272
3273                         yield f
3274
3275     def _extract_storyboard(self, player_responses, duration):
3276         spec = get_first(
3277             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3278         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3279         if not base_url:
3280             return
3281         L = len(spec) - 1
3282         for i, args in enumerate(spec):
3283             args = args.split('#')
3284             counts = list(map(int_or_none, args[:5]))
3285             if len(args) != 8 or not all(counts):
3286                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3287                 continue
3288             width, height, frame_count, cols, rows = counts
3289             N, sigh = args[6:]
3290
3291             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3292             fragment_count = frame_count / (cols * rows)
3293             fragment_duration = duration / fragment_count
3294             yield {
3295                 'format_id': f'sb{i}',
3296                 'format_note': 'storyboard',
3297                 'ext': 'mhtml',
3298                 'protocol': 'mhtml',
3299                 'acodec': 'none',
3300                 'vcodec': 'none',
3301                 'url': url,
3302                 'width': width,
3303                 'height': height,
3304                 'fragments': [{
3305                     'url': url.replace('$M', str(j)),
3306                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3307                 } for j in range(math.ceil(fragment_count))],
3308             }
3309
3310     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3311         webpage = None
3312         if 'webpage' not in self._configuration_arg('player_skip'):
3313             webpage = self._download_webpage(
3314                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3315
3316         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3317
3318         player_responses, player_url = self._extract_player_responses(
3319             self._get_requested_clients(url, smuggled_data),
3320             video_id, webpage, master_ytcfg)
3321
3322         return webpage, master_ytcfg, player_responses, player_url
3323
3324     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3325         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3326         is_live = get_first(video_details, 'isLive')
3327         if is_live is None:
3328             is_live = get_first(live_broadcast_details, 'isLiveNow')
3329
3330         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3331         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3332
3333         return live_broadcast_details, is_live, streaming_data, formats
3334
3335     def _real_extract(self, url):
3336         url, smuggled_data = unsmuggle_url(url, {})
3337         video_id = self._match_id(url)
3338
3339         base_url = self.http_scheme() + '//www.youtube.com/'
3340         webpage_url = base_url + 'watch?v=' + video_id
3341
3342         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3343
3344         playability_statuses = traverse_obj(
3345             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3346
3347         trailer_video_id = get_first(
3348             playability_statuses,
3349             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3350             expected_type=str)
3351         if trailer_video_id:
3352             return self.url_result(
3353                 trailer_video_id, self.ie_key(), trailer_video_id)
3354
3355         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3356                        if webpage else (lambda x: None))
3357
3358         video_details = traverse_obj(
3359             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3360         microformats = traverse_obj(
3361             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3362             expected_type=dict, default=[])
3363         video_title = (
3364             get_first(video_details, 'title')
3365             or self._get_text(microformats, (..., 'title'))
3366             or search_meta(['og:title', 'twitter:title', 'title']))
3367         video_description = get_first(video_details, 'shortDescription')
3368
3369         multifeed_metadata_list = get_first(
3370             player_responses,
3371             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3372             expected_type=str)
3373         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3374             if self.get_param('noplaylist'):
3375                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3376             else:
3377                 entries = []
3378                 feed_ids = []
3379                 for feed in multifeed_metadata_list.split(','):
3380                     # Unquote should take place before split on comma (,) since textual
3381                     # fields may contain comma as well (see
3382                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3383                     feed_data = compat_parse_qs(
3384                         compat_urllib_parse_unquote_plus(feed))
3385
3386                     def feed_entry(name):
3387                         return try_get(
3388                             feed_data, lambda x: x[name][0], compat_str)
3389
3390                     feed_id = feed_entry('id')
3391                     if not feed_id:
3392                         continue
3393                     feed_title = feed_entry('title')
3394                     title = video_title
3395                     if feed_title:
3396                         title += ' (%s)' % feed_title
3397                     entries.append({
3398                         '_type': 'url_transparent',
3399                         'ie_key': 'Youtube',
3400                         'url': smuggle_url(
3401                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3402                             {'force_singlefeed': True}),
3403                         'title': title,
3404                     })
3405                     feed_ids.append(feed_id)
3406                 self.to_screen(
3407                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3408                     % (', '.join(feed_ids), video_id))
3409                 return self.playlist_result(
3410                     entries, video_id, video_title, video_description)
3411
3412         duration = int_or_none(
3413             get_first(video_details, 'lengthSeconds')
3414             or get_first(microformats, 'lengthSeconds')
3415             or parse_duration(search_meta('duration'))) or None
3416
3417         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3418             video_id, microformats, video_details, player_responses, player_url, duration)
3419
3420         if not formats:
3421             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3422                 self.report_drm(video_id)
3423             pemr = get_first(
3424                 playability_statuses,
3425                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3426             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3427             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3428             if subreason:
3429                 if subreason == 'The uploader has not made this video available in your country.':
3430                     countries = get_first(microformats, 'availableCountries')
3431                     if not countries:
3432                         regions_allowed = search_meta('regionsAllowed')
3433                         countries = regions_allowed.split(',') if regions_allowed else None
3434                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3435                 reason += f'. {subreason}'
3436             if reason:
3437                 self.raise_no_formats(reason, expected=True)
3438
3439         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3440         if not keywords and webpage:
3441             keywords = [
3442                 unescapeHTML(m.group('content'))
3443                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3444         for keyword in keywords:
3445             if keyword.startswith('yt:stretch='):
3446                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3447                 if mobj:
3448                     # NB: float is intentional for forcing float division
3449                     w, h = (float(v) for v in mobj.groups())
3450                     if w > 0 and h > 0:
3451                         ratio = w / h
3452                         for f in formats:
3453                             if f.get('vcodec') != 'none':
3454                                 f['stretched_ratio'] = ratio
3455                         break
3456         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3457         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3458         if thumbnail_url:
3459             thumbnails.append({
3460                 'url': thumbnail_url,
3461             })
3462         original_thumbnails = thumbnails.copy()
3463
3464         # The best resolution thumbnails sometimes does not appear in the webpage
3465         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3466         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3467         thumbnail_names = [
3468             # While the *1,*2,*3 thumbnails are just below their correspnding "*default" variants
3469             # in resolution, these are not the custom thumbnail. So de-prioritize them
3470             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3471             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3472         ]
3473         n_thumbnail_names = len(thumbnail_names)
3474         thumbnails.extend({
3475             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3476                 video_id=video_id, name=name, ext=ext,
3477                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3478         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3479         for thumb in thumbnails:
3480             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3481             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3482         self._remove_duplicate_formats(thumbnails)
3483         self._downloader._sort_thumbnails(original_thumbnails)
3484
3485         category = get_first(microformats, 'category') or search_meta('genre')
3486         channel_id = str_or_none(
3487             get_first(video_details, 'channelId')
3488             or get_first(microformats, 'externalChannelId')
3489             or search_meta('channelId'))
3490         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3491
3492         live_content = get_first(video_details, 'isLiveContent')
3493         is_upcoming = get_first(video_details, 'isUpcoming')
3494         if is_live is None:
3495             if is_upcoming or live_content is False:
3496                 is_live = False
3497         if is_upcoming is None and (live_content or is_live):
3498             is_upcoming = False
3499         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3500         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3501         if not duration and live_end_time and live_start_time:
3502             duration = live_end_time - live_start_time
3503
3504         if is_live and self.get_param('live_from_start'):
3505             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3506
3507         formats.extend(self._extract_storyboard(player_responses, duration))
3508
3509         # Source is given priority since formats that throttle are given lower source_preference
3510         # When throttling issue is fully fixed, remove this
3511         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3512
3513         info = {
3514             'id': video_id,
3515             'title': video_title,
3516             'formats': formats,
3517             'thumbnails': thumbnails,
3518             # The best thumbnail that we are sure exists. Prevents unnecessary
3519             # URL checking if user don't care about getting the best possible thumbnail
3520             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3521             'description': video_description,
3522             'uploader': get_first(video_details, 'author'),
3523             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3524             'uploader_url': owner_profile_url,
3525             'channel_id': channel_id,
3526             'channel_url': format_field(channel_id, template='https://www.youtube.com/channel/%s'),
3527             'duration': duration,
3528             'view_count': int_or_none(
3529                 get_first((video_details, microformats), (..., 'viewCount'))
3530                 or search_meta('interactionCount')),
3531             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3532             'age_limit': 18 if (
3533                 get_first(microformats, 'isFamilySafe') is False
3534                 or search_meta('isFamilyFriendly') == 'false'
3535                 or search_meta('og:restrictions:age') == '18+') else 0,
3536             'webpage_url': webpage_url,
3537             'categories': [category] if category else None,
3538             'tags': keywords,
3539             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3540             'is_live': is_live,
3541             'was_live': (False if is_live or is_upcoming or live_content is False
3542                          else None if is_live is None or is_upcoming is None
3543                          else live_content),
3544             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3545             'release_timestamp': live_start_time,
3546         }
3547
3548         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3549         if pctr:
3550             def get_lang_code(track):
3551                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3552                         or track.get('languageCode'))
3553
3554             # Converted into dicts to remove duplicates
3555             captions = {
3556                 get_lang_code(sub): sub
3557                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3558             translation_languages = {
3559                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3560                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3561
3562             def process_language(container, base_url, lang_code, sub_name, query):
3563                 lang_subs = container.setdefault(lang_code, [])
3564                 for fmt in self._SUBTITLE_FORMATS:
3565                     query.update({
3566                         'fmt': fmt,
3567                     })
3568                     lang_subs.append({
3569                         'ext': fmt,
3570                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3571                         'name': sub_name,
3572                     })
3573
3574             subtitles, automatic_captions = {}, {}
3575             for lang_code, caption_track in captions.items():
3576                 base_url = caption_track.get('baseUrl')
3577                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3578                 if not base_url:
3579                     continue
3580                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3581                 if caption_track.get('kind') != 'asr':
3582                     if not lang_code:
3583                         continue
3584                     process_language(
3585                         subtitles, base_url, lang_code, lang_name, {})
3586                     if not caption_track.get('isTranslatable'):
3587                         continue
3588                 for trans_code, trans_name in translation_languages.items():
3589                     if not trans_code:
3590                         continue
3591                     orig_trans_code = trans_code
3592                     if caption_track.get('kind') != 'asr':
3593                         if 'translated_subs' in self._configuration_arg('skip'):
3594                             continue
3595                         trans_code += f'-{lang_code}'
3596                         trans_name += format_field(lang_name, template=' from %s')
3597                     # Add an "-orig" label to the original language so that it can be distinguished.
3598                     # The subs are returned without "-orig" as well for compatibility
3599                     if lang_code == f'a-{orig_trans_code}':
3600                         process_language(
3601                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3602                     # Setting tlang=lang returns damaged subtitles.
3603                     process_language(automatic_captions, base_url, trans_code, trans_name,
3604                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3605             info['automatic_captions'] = automatic_captions
3606             info['subtitles'] = subtitles
3607
3608         parsed_url = compat_urllib_parse_urlparse(url)
3609         for component in [parsed_url.fragment, parsed_url.query]:
3610             query = compat_parse_qs(component)
3611             for k, v in query.items():
3612                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3613                     d_k += '_time'
3614                     if d_k not in info and k in s_ks:
3615                         info[d_k] = parse_duration(query[k][0])
3616
3617         # Youtube Music Auto-generated description
3618         if video_description:
3619             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
3620             if mobj:
3621                 release_year = mobj.group('release_year')
3622                 release_date = mobj.group('release_date')
3623                 if release_date:
3624                     release_date = release_date.replace('-', '')
3625                     if not release_year:
3626                         release_year = release_date[:4]
3627                 info.update({
3628                     'album': mobj.group('album'.strip()),
3629                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3630                     'track': mobj.group('track').strip(),
3631                     'release_date': release_date,
3632                     'release_year': int_or_none(release_year),
3633                 })
3634
3635         initial_data = None
3636         if webpage:
3637             initial_data = self._extract_yt_initial_variable(
3638                 webpage, self._YT_INITIAL_DATA_RE, video_id,
3639                 'yt initial data')
3640         if not initial_data:
3641             query = {'videoId': video_id}
3642             query.update(self._get_checkok_params())
3643             initial_data = self._extract_response(
3644                 item_id=video_id, ep='next', fatal=False,
3645                 ytcfg=master_ytcfg, query=query,
3646                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3647                 note='Downloading initial data API JSON')
3648
3649         try:  # This will error if there is no livechat
3650             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3651         except (KeyError, IndexError, TypeError):
3652             pass
3653         else:
3654             info.setdefault('subtitles', {})['live_chat'] = [{
3655                 'url': f'https://www.youtube.com/watch?v={video_id}',  # url is needed to set cookies
3656                 'video_id': video_id,
3657                 'ext': 'json',
3658                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3659             }]
3660
3661         if initial_data:
3662             info['chapters'] = (
3663                 self._extract_chapters_from_json(initial_data, duration)
3664                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3665                 or None)
3666
3667         contents = traverse_obj(
3668             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3669             expected_type=list, default=[])
3670
3671         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3672         if vpir:
3673             stl = vpir.get('superTitleLink')
3674             if stl:
3675                 stl = self._get_text(stl)
3676                 if try_get(
3677                         vpir,
3678                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3679                     info['location'] = stl
3680                 else:
3681                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3682                     if mobj:
3683                         info.update({
3684                             'series': mobj.group(1),
3685                             'season_number': int(mobj.group(2)),
3686                             'episode_number': int(mobj.group(3)),
3687                         })
3688             for tlb in (try_get(
3689                     vpir,
3690                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3691                     list) or []):
3692                 tbr = tlb.get('toggleButtonRenderer') or {}
3693                 for getter, regex in [(
3694                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3695                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3696                             lambda x: x['accessibility'],
3697                             lambda x: x['accessibilityData']['accessibilityData'],
3698                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3699                     label = (try_get(tbr, getter, dict) or {}).get('label')
3700                     if label:
3701                         mobj = re.match(regex, label)
3702                         if mobj:
3703                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3704                             break
3705             sbr_tooltip = try_get(
3706                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3707             if sbr_tooltip:
3708                 like_count, dislike_count = sbr_tooltip.split(' / ')
3709                 info.update({
3710                     'like_count': str_to_int(like_count),
3711                     'dislike_count': str_to_int(dislike_count),
3712                 })
3713         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3714         if vsir:
3715             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3716             info.update({
3717                 'channel': self._get_text(vor, 'title'),
3718                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3719
3720             rows = try_get(
3721                 vsir,
3722                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3723                 list) or []
3724             multiple_songs = False
3725             for row in rows:
3726                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3727                     multiple_songs = True
3728                     break
3729             for row in rows:
3730                 mrr = row.get('metadataRowRenderer') or {}
3731                 mrr_title = mrr.get('title')
3732                 if not mrr_title:
3733                     continue
3734                 mrr_title = self._get_text(mrr, 'title')
3735                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3736                 if mrr_title == 'License':
3737                     info['license'] = mrr_contents_text
3738                 elif not multiple_songs:
3739                     if mrr_title == 'Album':
3740                         info['album'] = mrr_contents_text
3741                     elif mrr_title == 'Artist':
3742                         info['artist'] = mrr_contents_text
3743                     elif mrr_title == 'Song':
3744                         info['track'] = mrr_contents_text
3745
3746         fallbacks = {
3747             'channel': 'uploader',
3748             'channel_id': 'uploader_id',
3749             'channel_url': 'uploader_url',
3750         }
3751
3752         # The upload date for scheduled, live and past live streams / premieres in microformats
3753         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3754         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3755         upload_date = (
3756             unified_strdate(get_first(microformats, 'uploadDate'))
3757             or unified_strdate(search_meta('uploadDate')))
3758         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3759             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3760         info['upload_date'] = upload_date
3761
3762         for to, frm in fallbacks.items():
3763             if not info.get(to):
3764                 info[to] = info.get(frm)
3765
3766         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3767             v = info.get(s_k)
3768             if v:
3769                 info[d_k] = v
3770
3771         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3772         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3773         is_membersonly = None
3774         is_premium = None
3775         if initial_data and is_private is not None:
3776             is_membersonly = False
3777             is_premium = False
3778             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3779             badge_labels = set()
3780             for content in contents:
3781                 if not isinstance(content, dict):
3782                     continue
3783                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3784             for badge_label in badge_labels:
3785                 if badge_label.lower() == 'members only':
3786                     is_membersonly = True
3787                 elif badge_label.lower() == 'premium':
3788                     is_premium = True
3789                 elif badge_label.lower() == 'unlisted':
3790                     is_unlisted = True
3791
3792         info['availability'] = self._availability(
3793             is_private=is_private,
3794             needs_premium=is_premium,
3795             needs_subscription=is_membersonly,
3796             needs_auth=info['age_limit'] >= 18,
3797             is_unlisted=None if is_private is None else is_unlisted)
3798
3799         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3800
3801         self.mark_watched(video_id, player_responses)
3802
3803         return info
3804
3805
3806 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3807
3808     @staticmethod
3809     def passthrough_smuggled_data(func):
3810         def _smuggle(entries, smuggled_data):
3811             for entry in entries:
3812                 # TODO: Convert URL to music.youtube instead.
3813                 # Do we need to passthrough any other smuggled_data?
3814                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3815                 yield entry
3816
3817         @functools.wraps(func)
3818         def wrapper(self, url):
3819             url, smuggled_data = unsmuggle_url(url, {})
3820             if self.is_music_url(url):
3821                 smuggled_data['is_music_url'] = True
3822             info_dict = func(self, url, smuggled_data)
3823             if smuggled_data and info_dict.get('entries'):
3824                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3825             return info_dict
3826         return wrapper
3827
3828     def _extract_channel_id(self, webpage):
3829         channel_id = self._html_search_meta(
3830             'channelId', webpage, 'channel id', default=None)
3831         if channel_id:
3832             return channel_id
3833         channel_url = self._html_search_meta(
3834             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3835              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3836              'twitter:app:url:googleplay'), webpage, 'channel url')
3837         return self._search_regex(
3838             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3839             channel_url, 'channel id')
3840
3841     @staticmethod
3842     def _extract_basic_item_renderer(item):
3843         # Modified from _extract_grid_item_renderer
3844         known_basic_renderers = (
3845             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3846         )
3847         for key, renderer in item.items():
3848             if not isinstance(renderer, dict):
3849                 continue
3850             elif key in known_basic_renderers:
3851                 return renderer
3852             elif key.startswith('grid') and key.endswith('Renderer'):
3853                 return renderer
3854
3855     def _grid_entries(self, grid_renderer):
3856         for item in grid_renderer['items']:
3857             if not isinstance(item, dict):
3858                 continue
3859             renderer = self._extract_basic_item_renderer(item)
3860             if not isinstance(renderer, dict):
3861                 continue
3862             title = self._get_text(renderer, 'title')
3863
3864             # playlist
3865             playlist_id = renderer.get('playlistId')
3866             if playlist_id:
3867                 yield self.url_result(
3868                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3869                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3870                     video_title=title)
3871                 continue
3872             # video
3873             video_id = renderer.get('videoId')
3874             if video_id:
3875                 yield self._extract_video(renderer)
3876                 continue
3877             # channel
3878             channel_id = renderer.get('channelId')
3879             if channel_id:
3880                 yield self.url_result(
3881                     'https://www.youtube.com/channel/%s' % channel_id,
3882                     ie=YoutubeTabIE.ie_key(), video_title=title)
3883                 continue
3884             # generic endpoint URL support
3885             ep_url = urljoin('https://www.youtube.com/', try_get(
3886                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3887                 compat_str))
3888             if ep_url:
3889                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3890                     if ie.suitable(ep_url):
3891                         yield self.url_result(
3892                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3893                         break
3894
3895     def _music_reponsive_list_entry(self, renderer):
3896         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3897         if video_id:
3898             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3899                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3900         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3901         if playlist_id:
3902             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3903             if video_id:
3904                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3905                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3906             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3907                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3908         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3909         if browse_id:
3910             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3911                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3912
3913     def _shelf_entries_from_content(self, shelf_renderer):
3914         content = shelf_renderer.get('content')
3915         if not isinstance(content, dict):
3916             return
3917         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3918         if renderer:
3919             # TODO: add support for nested playlists so each shelf is processed
3920             # as separate playlist
3921             # TODO: this includes only first N items
3922             yield from self._grid_entries(renderer)
3923         renderer = content.get('horizontalListRenderer')
3924         if renderer:
3925             # TODO
3926             pass
3927
3928     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3929         ep = try_get(
3930             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3931             compat_str)
3932         shelf_url = urljoin('https://www.youtube.com', ep)
3933         if shelf_url:
3934             # Skipping links to another channels, note that checking for
3935             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3936             # will not work
3937             if skip_channels and '/channels?' in shelf_url:
3938                 return
3939             title = self._get_text(shelf_renderer, 'title')
3940             yield self.url_result(shelf_url, video_title=title)
3941         # Shelf may not contain shelf URL, fallback to extraction from content
3942         yield from self._shelf_entries_from_content(shelf_renderer)
3943
3944     def _playlist_entries(self, video_list_renderer):
3945         for content in video_list_renderer['contents']:
3946             if not isinstance(content, dict):
3947                 continue
3948             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3949             if not isinstance(renderer, dict):
3950                 continue
3951             video_id = renderer.get('videoId')
3952             if not video_id:
3953                 continue
3954             yield self._extract_video(renderer)
3955
3956     def _rich_entries(self, rich_grid_renderer):
3957         renderer = try_get(
3958             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3959         video_id = renderer.get('videoId')
3960         if not video_id:
3961             return
3962         yield self._extract_video(renderer)
3963
3964     def _video_entry(self, video_renderer):
3965         video_id = video_renderer.get('videoId')
3966         if video_id:
3967             return self._extract_video(video_renderer)
3968
3969     def _hashtag_tile_entry(self, hashtag_tile_renderer):
3970         url = urljoin('https://youtube.com', traverse_obj(
3971             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
3972         if url:
3973             return self.url_result(
3974                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
3975
3976     def _post_thread_entries(self, post_thread_renderer):
3977         post_renderer = try_get(
3978             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
3979         if not post_renderer:
3980             return
3981         # video attachment
3982         video_renderer = try_get(
3983             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
3984         video_id = video_renderer.get('videoId')
3985         if video_id:
3986             entry = self._extract_video(video_renderer)
3987             if entry:
3988                 yield entry
3989         # playlist attachment
3990         playlist_id = try_get(
3991             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
3992         if playlist_id:
3993             yield self.url_result(
3994                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
3995                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3996         # inline video links
3997         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
3998         for run in runs:
3999             if not isinstance(run, dict):
4000                 continue
4001             ep_url = try_get(
4002                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
4003             if not ep_url:
4004                 continue
4005             if not YoutubeIE.suitable(ep_url):
4006                 continue
4007             ep_video_id = YoutubeIE._match_id(ep_url)
4008             if video_id == ep_video_id:
4009                 continue
4010             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4011
4012     def _post_thread_continuation_entries(self, post_thread_continuation):
4013         contents = post_thread_continuation.get('contents')
4014         if not isinstance(contents, list):
4015             return
4016         for content in contents:
4017             renderer = content.get('backstagePostThreadRenderer')
4018             if not isinstance(renderer, dict):
4019                 continue
4020             yield from self._post_thread_entries(renderer)
4021
4022     r''' # unused
4023     def _rich_grid_entries(self, contents):
4024         for content in contents:
4025             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4026             if video_renderer:
4027                 entry = self._video_entry(video_renderer)
4028                 if entry:
4029                     yield entry
4030     '''
4031
4032     def _extract_entries(self, parent_renderer, continuation_list):
4033         # continuation_list is modified in-place with continuation_list = [continuation_token]
4034         continuation_list[:] = [None]
4035         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4036         for content in contents:
4037             if not isinstance(content, dict):
4038                 continue
4039             is_renderer = traverse_obj(
4040                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4041                 expected_type=dict)
4042             if not is_renderer:
4043                 renderer = content.get('richItemRenderer')
4044                 if renderer:
4045                     for entry in self._rich_entries(renderer):
4046                         yield entry
4047                     continuation_list[0] = self._extract_continuation(parent_renderer)
4048                 continue
4049             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4050             for isr_content in isr_contents:
4051                 if not isinstance(isr_content, dict):
4052                     continue
4053
4054                 known_renderers = {
4055                     'playlistVideoListRenderer': self._playlist_entries,
4056                     'gridRenderer': self._grid_entries,
4057                     'reelShelfRenderer': self._grid_entries,
4058                     'shelfRenderer': self._shelf_entries,
4059                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4060                     'backstagePostThreadRenderer': self._post_thread_entries,
4061                     'videoRenderer': lambda x: [self._video_entry(x)],
4062                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4063                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4064                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4065                 }
4066                 for key, renderer in isr_content.items():
4067                     if key not in known_renderers:
4068                         continue
4069                     for entry in known_renderers[key](renderer):
4070                         if entry:
4071                             yield entry
4072                     continuation_list[0] = self._extract_continuation(renderer)
4073                     break
4074
4075             if not continuation_list[0]:
4076                 continuation_list[0] = self._extract_continuation(is_renderer)
4077
4078         if not continuation_list[0]:
4079             continuation_list[0] = self._extract_continuation(parent_renderer)
4080
4081     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4082         continuation_list = [None]
4083         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4084         tab_content = try_get(tab, lambda x: x['content'], dict)
4085         if not tab_content:
4086             return
4087         parent_renderer = (
4088             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4089             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4090         yield from extract_entries(parent_renderer)
4091         continuation = continuation_list[0]
4092
4093         for page_num in itertools.count(1):
4094             if not continuation:
4095                 break
4096             headers = self.generate_api_headers(
4097                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4098             response = self._extract_response(
4099                 item_id=f'{item_id} page {page_num}',
4100                 query=continuation, headers=headers, ytcfg=ytcfg,
4101                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4102
4103             if not response:
4104                 break
4105             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4106             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4107             visitor_data = self._extract_visitor_data(response) or visitor_data
4108
4109             known_continuation_renderers = {
4110                 'playlistVideoListContinuation': self._playlist_entries,
4111                 'gridContinuation': self._grid_entries,
4112                 'itemSectionContinuation': self._post_thread_continuation_entries,
4113                 'sectionListContinuation': extract_entries,  # for feeds
4114             }
4115             continuation_contents = try_get(
4116                 response, lambda x: x['continuationContents'], dict) or {}
4117             continuation_renderer = None
4118             for key, value in continuation_contents.items():
4119                 if key not in known_continuation_renderers:
4120                     continue
4121                 continuation_renderer = value
4122                 continuation_list = [None]
4123                 yield from known_continuation_renderers[key](continuation_renderer)
4124                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4125                 break
4126             if continuation_renderer:
4127                 continue
4128
4129             known_renderers = {
4130                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4131                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4132                 'gridVideoRenderer': (self._grid_entries, 'items'),
4133                 'gridChannelRenderer': (self._grid_entries, 'items'),
4134                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4135                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4136                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4137                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4138             }
4139             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4140             continuation_items = try_get(
4141                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4142             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4143             video_items_renderer = None
4144             for key, value in continuation_item.items():
4145                 if key not in known_renderers:
4146                     continue
4147                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4148                 continuation_list = [None]
4149                 yield from known_renderers[key][0](video_items_renderer)
4150                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4151                 break
4152             if video_items_renderer:
4153                 continue
4154             break
4155
4156     @staticmethod
4157     def _extract_selected_tab(tabs, fatal=True):
4158         for tab in tabs:
4159             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4160             if renderer.get('selected') is True:
4161                 return renderer
4162         else:
4163             if fatal:
4164                 raise ExtractorError('Unable to find selected tab')
4165
4166     def _extract_uploader(self, data):
4167         uploader = {}
4168         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4169         owner = try_get(
4170             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4171         if owner:
4172             owner_text = owner.get('text')
4173             uploader['uploader'] = self._search_regex(
4174                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4175             uploader['uploader_id'] = try_get(
4176                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
4177             uploader['uploader_url'] = urljoin(
4178                 'https://www.youtube.com/',
4179                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
4180         return {k: v for k, v in uploader.items() if v is not None}
4181
4182     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4183         playlist_id = title = description = channel_url = channel_name = channel_id = None
4184         tags = []
4185
4186         selected_tab = self._extract_selected_tab(tabs)
4187         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4188         renderer = try_get(
4189             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4190         if renderer:
4191             channel_name = renderer.get('title')
4192             channel_url = renderer.get('channelUrl')
4193             channel_id = renderer.get('externalId')
4194         else:
4195             renderer = try_get(
4196                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4197
4198         if renderer:
4199             title = renderer.get('title')
4200             description = renderer.get('description', '')
4201             playlist_id = channel_id
4202             tags = renderer.get('keywords', '').split()
4203
4204         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4205         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4206         def _get_uncropped(url):
4207             return url_or_none((url or '').split('=')[0] + '=s0')
4208
4209         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4210         if avatar_thumbnails:
4211             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4212             if uncropped_avatar:
4213                 avatar_thumbnails.append({
4214                     'url': uncropped_avatar,
4215                     'id': 'avatar_uncropped',
4216                     'preference': 1
4217                 })
4218
4219         channel_banners = self._extract_thumbnails(
4220             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4221         for banner in channel_banners:
4222             banner['preference'] = -10
4223
4224         if channel_banners:
4225             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4226             if uncropped_banner:
4227                 channel_banners.append({
4228                     'url': uncropped_banner,
4229                     'id': 'banner_uncropped',
4230                     'preference': -5
4231                 })
4232
4233         primary_thumbnails = self._extract_thumbnails(
4234             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4235
4236         if playlist_id is None:
4237             playlist_id = item_id
4238
4239         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4240         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4241         if title is None:
4242             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4243         title += format_field(selected_tab, 'title', ' - %s')
4244         title += format_field(selected_tab, 'expandedText', ' - %s')
4245
4246         metadata = {
4247             'playlist_id': playlist_id,
4248             'playlist_title': title,
4249             'playlist_description': description,
4250             'uploader': channel_name,
4251             'uploader_id': channel_id,
4252             'uploader_url': channel_url,
4253             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4254             'tags': tags,
4255             'view_count': self._get_count(playlist_stats, 1),
4256             'availability': self._extract_availability(data),
4257             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4258             'playlist_count': self._get_count(playlist_stats, 0),
4259             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4260         }
4261         if not channel_id:
4262             metadata.update(self._extract_uploader(data))
4263         metadata.update({
4264             'channel': metadata['uploader'],
4265             'channel_id': metadata['uploader_id'],
4266             'channel_url': metadata['uploader_url']})
4267         return self.playlist_result(
4268             self._entries(
4269                 selected_tab, playlist_id, ytcfg,
4270                 self._extract_account_syncid(ytcfg, data),
4271                 self._extract_visitor_data(data, ytcfg)),
4272             **metadata)
4273
4274     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4275         first_id = last_id = response = None
4276         for page_num in itertools.count(1):
4277             videos = list(self._playlist_entries(playlist))
4278             if not videos:
4279                 return
4280             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4281             if start >= len(videos):
4282                 return
4283             yield from videos[start:]
4284             first_id = first_id or videos[0]['id']
4285             last_id = videos[-1]['id']
4286             watch_endpoint = try_get(
4287                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4288             headers = self.generate_api_headers(
4289                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4290                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4291             query = {
4292                 'playlistId': playlist_id,
4293                 'videoId': watch_endpoint.get('videoId') or last_id,
4294                 'index': watch_endpoint.get('index') or len(videos),
4295                 'params': watch_endpoint.get('params') or 'OAE%3D'
4296             }
4297             response = self._extract_response(
4298                 item_id='%s page %d' % (playlist_id, page_num),
4299                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4300                 check_get_keys='contents'
4301             )
4302             playlist = try_get(
4303                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4304
4305     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4306         title = playlist.get('title') or try_get(
4307             data, lambda x: x['titleText']['simpleText'], compat_str)
4308         playlist_id = playlist.get('playlistId') or item_id
4309
4310         # Delegating everything except mix playlists to regular tab-based playlist URL
4311         playlist_url = urljoin(url, try_get(
4312             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4313             compat_str))
4314
4315         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4316         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4317         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4318
4319         if playlist_url and playlist_url != url and not is_known_unviewable:
4320             return self.url_result(
4321                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4322                 video_title=title)
4323
4324         return self.playlist_result(
4325             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4326             playlist_id=playlist_id, playlist_title=title)
4327
4328     def _extract_availability(self, data):
4329         """
4330         Gets the availability of a given playlist/tab.
4331         Note: Unless YouTube tells us explicitly, we do not assume it is public
4332         @param data: response
4333         """
4334         is_private = is_unlisted = None
4335         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4336         badge_labels = self._extract_badges(renderer)
4337
4338         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4339         privacy_dropdown_entries = try_get(
4340             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4341         for renderer_dict in privacy_dropdown_entries:
4342             is_selected = try_get(
4343                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4344             if not is_selected:
4345                 continue
4346             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4347             if label:
4348                 badge_labels.add(label.lower())
4349                 break
4350
4351         for badge_label in badge_labels:
4352             if badge_label == 'unlisted':
4353                 is_unlisted = True
4354             elif badge_label == 'private':
4355                 is_private = True
4356             elif badge_label == 'public':
4357                 is_unlisted = is_private = False
4358         return self._availability(is_private, False, False, False, is_unlisted)
4359
4360     @staticmethod
4361     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4362         sidebar_renderer = try_get(
4363             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4364         for item in sidebar_renderer:
4365             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4366             if renderer:
4367                 return renderer
4368
4369     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4370         """
4371         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4372         """
4373         browse_id = params = None
4374         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4375         if not renderer:
4376             return
4377         menu_renderer = try_get(
4378             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4379         for menu_item in menu_renderer:
4380             if not isinstance(menu_item, dict):
4381                 continue
4382             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4383             text = try_get(
4384                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4385             if not text or text.lower() != 'show unavailable videos':
4386                 continue
4387             browse_endpoint = try_get(
4388                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4389             browse_id = browse_endpoint.get('browseId')
4390             params = browse_endpoint.get('params')
4391             break
4392
4393         headers = self.generate_api_headers(
4394             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4395             visitor_data=self._extract_visitor_data(data, ytcfg))
4396         query = {
4397             'params': params or 'wgYCCAA=',
4398             'browseId': browse_id or 'VL%s' % item_id
4399         }
4400         return self._extract_response(
4401             item_id=item_id, headers=headers, query=query,
4402             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4403             note='Downloading API JSON with unavailable videos')
4404
4405     @property
4406     def skip_webpage(self):
4407         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4408
4409     def _extract_webpage(self, url, item_id, fatal=True):
4410         retries = self.get_param('extractor_retries', 3)
4411         count = -1
4412         webpage = data = last_error = None
4413         while count < retries:
4414             count += 1
4415             # Sometimes youtube returns a webpage with incomplete ytInitialData
4416             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4417             if last_error:
4418                 self.report_warning('%s. Retrying ...' % last_error)
4419             try:
4420                 webpage = self._download_webpage(
4421                     url, item_id,
4422                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4423                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4424             except ExtractorError as e:
4425                 if isinstance(e.cause, network_exceptions):
4426                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4427                         last_error = error_to_compat_str(e.cause or e.msg)
4428                         if count < retries:
4429                             continue
4430                 if fatal:
4431                     raise
4432                 self.report_warning(error_to_compat_str(e))
4433                 break
4434             else:
4435                 try:
4436                     self._extract_and_report_alerts(data)
4437                 except ExtractorError as e:
4438                     if fatal:
4439                         raise
4440                     self.report_warning(error_to_compat_str(e))
4441                     break
4442
4443                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4444                     break
4445
4446                 last_error = 'Incomplete yt initial data received'
4447                 if count >= retries:
4448                     if fatal:
4449                         raise ExtractorError(last_error)
4450                     self.report_warning(last_error)
4451                     break
4452
4453         return webpage, data
4454
4455     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4456         """Use if failed to extract ytcfg (and data) from initial webpage"""
4457         if not ytcfg and self.is_authenticated:
4458             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4459             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4460                 raise ExtractorError(
4461                     f'{msg}. If you are not downloading private content, or '
4462                     'your cookies are only for the first account and channel,'
4463                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4464                     expected=True)
4465             self.report_warning(msg, only_once=True)
4466
4467     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4468         data = None
4469         if not self.skip_webpage:
4470             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4471             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4472             # Reject webpage data if redirected to home page without explicitly requesting
4473             selected_tab = self._extract_selected_tab(traverse_obj(
4474                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4475             if (url != 'https://www.youtube.com/feed/recommended'
4476                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4477                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4478                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4479                 if fatal:
4480                     raise ExtractorError(msg, expected=True)
4481                 self.report_warning(msg, only_once=True)
4482         if not data:
4483             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4484             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4485         return data, ytcfg
4486
4487     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4488         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4489         resolve_response = self._extract_response(
4490             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4491             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4492         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4493         for ep_key, ep in endpoints.items():
4494             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4495             if params:
4496                 return self._extract_response(
4497                     item_id=item_id, query=params, ep=ep, headers=headers,
4498                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4499                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4500         err_note = 'Failed to resolve url (does the playlist exist?)'
4501         if fatal:
4502             raise ExtractorError(err_note, expected=True)
4503         self.report_warning(err_note, item_id)
4504
4505     _SEARCH_PARAMS = None
4506
4507     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4508         data = {'query': query}
4509         if params is NO_DEFAULT:
4510             params = self._SEARCH_PARAMS
4511         if params:
4512             data['params'] = params
4513
4514         content_keys = (
4515             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4516             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4517             # ytmusic search
4518             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4519             ('continuationContents', ),
4520         )
4521         display_id = f'query "{query}"'
4522         check_get_keys = tuple({keys[0] for keys in content_keys})
4523         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4524         self._report_playlist_authcheck(ytcfg, fatal=False)
4525
4526         continuation_list = [None]
4527         search = None
4528         for page_num in itertools.count(1):
4529             data.update(continuation_list[0] or {})
4530             headers = self.generate_api_headers(
4531                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4532             search = self._extract_response(
4533                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4534                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4535             slr_contents = traverse_obj(search, *content_keys)
4536             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4537             if not continuation_list[0]:
4538                 break
4539
4540
4541 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4542     IE_DESC = 'YouTube Tabs'
4543     _VALID_URL = r'''(?x:
4544         https?://
4545             (?:\w+\.)?
4546             (?:
4547                 youtube(?:kids)?\.com|
4548                 %(invidious)s
4549             )/
4550             (?:
4551                 (?P<channel_type>channel|c|user|browse)/|
4552                 (?P<not_channel>
4553                     feed/|hashtag/|
4554                     (?:playlist|watch)\?.*?\blist=
4555                 )|
4556                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4557             )
4558             (?P<id>[^/?\#&]+)
4559     )''' % {
4560         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4561         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4562     }
4563     IE_NAME = 'youtube:tab'
4564
4565     _TESTS = [{
4566         'note': 'playlists, multipage',
4567         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4568         'playlist_mincount': 94,
4569         'info_dict': {
4570             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4571             'title': 'Igor Kleiner - Playlists',
4572             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4573             'uploader': 'Igor Kleiner',
4574             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4575             'channel': 'Igor Kleiner',
4576             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4577             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4578             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4579             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4580             'channel_follower_count': int
4581         },
4582     }, {
4583         'note': 'playlists, multipage, different order',
4584         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4585         'playlist_mincount': 94,
4586         'info_dict': {
4587             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4588             'title': 'Igor Kleiner - Playlists',
4589             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4590             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4591             'uploader': 'Igor Kleiner',
4592             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4593             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4594             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4595             'channel': 'Igor Kleiner',
4596             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4597             'channel_follower_count': int
4598         },
4599     }, {
4600         'note': 'playlists, series',
4601         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4602         'playlist_mincount': 5,
4603         'info_dict': {
4604             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4605             'title': '3Blue1Brown - Playlists',
4606             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4607             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4608             'uploader': '3Blue1Brown',
4609             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4610             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4611             'channel': '3Blue1Brown',
4612             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4613             'tags': ['Mathematics'],
4614             'channel_follower_count': int
4615         },
4616     }, {
4617         'note': 'playlists, singlepage',
4618         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4619         'playlist_mincount': 4,
4620         'info_dict': {
4621             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4622             'title': 'ThirstForScience - Playlists',
4623             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4624             'uploader': 'ThirstForScience',
4625             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4626             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4627             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4628             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4629             'tags': 'count:13',
4630             'channel': 'ThirstForScience',
4631             'channel_follower_count': int
4632         }
4633     }, {
4634         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4635         'only_matching': True,
4636     }, {
4637         'note': 'basic, single video playlist',
4638         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4639         'info_dict': {
4640             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4641             'uploader': 'Sergey M.',
4642             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4643             'title': 'youtube-dl public playlist',
4644             'description': '',
4645             'tags': [],
4646             'view_count': int,
4647             'modified_date': '20201130',
4648             'channel': 'Sergey M.',
4649             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4650             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4651             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4652         },
4653         'playlist_count': 1,
4654     }, {
4655         'note': 'empty playlist',
4656         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4657         'info_dict': {
4658             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4659             'uploader': 'Sergey M.',
4660             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4661             'title': 'youtube-dl empty playlist',
4662             'tags': [],
4663             'channel': 'Sergey M.',
4664             'description': '',
4665             'modified_date': '20160902',
4666             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4667             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4668             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4669         },
4670         'playlist_count': 0,
4671     }, {
4672         'note': 'Home tab',
4673         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4674         'info_dict': {
4675             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4676             'title': 'lex will - Home',
4677             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4678             'uploader': 'lex will',
4679             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4680             'channel': 'lex will',
4681             'tags': ['bible', 'history', 'prophesy'],
4682             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4683             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4684             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4685             'channel_follower_count': int
4686         },
4687         'playlist_mincount': 2,
4688     }, {
4689         'note': 'Videos tab',
4690         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4691         'info_dict': {
4692             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4693             'title': 'lex will - Videos',
4694             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4695             'uploader': 'lex will',
4696             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4697             'tags': ['bible', 'history', 'prophesy'],
4698             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4699             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4700             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4701             'channel': 'lex will',
4702             'channel_follower_count': int
4703         },
4704         'playlist_mincount': 975,
4705     }, {
4706         'note': 'Videos tab, sorted by popular',
4707         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4708         'info_dict': {
4709             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4710             'title': 'lex will - Videos',
4711             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4712             'uploader': 'lex will',
4713             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4714             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4715             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4716             'channel': 'lex will',
4717             'tags': ['bible', 'history', 'prophesy'],
4718             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4719             'channel_follower_count': int
4720         },
4721         'playlist_mincount': 199,
4722     }, {
4723         'note': 'Playlists tab',
4724         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4725         'info_dict': {
4726             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4727             'title': 'lex will - Playlists',
4728             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4729             'uploader': 'lex will',
4730             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4731             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4732             'channel': 'lex will',
4733             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4734             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4735             'tags': ['bible', 'history', 'prophesy'],
4736             'channel_follower_count': int
4737         },
4738         'playlist_mincount': 17,
4739     }, {
4740         'note': 'Community tab',
4741         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4742         'info_dict': {
4743             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4744             'title': 'lex will - Community',
4745             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4746             'uploader': 'lex will',
4747             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4748             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4749             'channel': 'lex will',
4750             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4751             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4752             'tags': ['bible', 'history', 'prophesy'],
4753             'channel_follower_count': int
4754         },
4755         'playlist_mincount': 18,
4756     }, {
4757         'note': 'Channels tab',
4758         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4759         'info_dict': {
4760             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4761             'title': 'lex will - Channels',
4762             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4763             'uploader': 'lex will',
4764             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4765             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4766             'channel': 'lex will',
4767             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4768             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4769             'tags': ['bible', 'history', 'prophesy'],
4770             'channel_follower_count': int
4771         },
4772         'playlist_mincount': 12,
4773     }, {
4774         'note': 'Search tab',
4775         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4776         'playlist_mincount': 40,
4777         'info_dict': {
4778             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4779             'title': '3Blue1Brown - Search - linear algebra',
4780             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4781             'uploader': '3Blue1Brown',
4782             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4783             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4784             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4785             'tags': ['Mathematics'],
4786             'channel': '3Blue1Brown',
4787             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4788             'channel_follower_count': int
4789         },
4790     }, {
4791         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4792         'only_matching': True,
4793     }, {
4794         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4795         'only_matching': True,
4796     }, {
4797         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4798         'only_matching': True,
4799     }, {
4800         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4801         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4802         'info_dict': {
4803             'title': '29C3: Not my department',
4804             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4805             'uploader': 'Christiaan008',
4806             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4807             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4808             'tags': [],
4809             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4810             'view_count': int,
4811             'modified_date': '20150605',
4812             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4813             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4814             'channel': 'Christiaan008',
4815         },
4816         'playlist_count': 96,
4817     }, {
4818         'note': 'Large playlist',
4819         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4820         'info_dict': {
4821             'title': 'Uploads from Cauchemar',
4822             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4823             'uploader': 'Cauchemar',
4824             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4825             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4826             'tags': [],
4827             'modified_date': r're:\d{8}',
4828             'channel': 'Cauchemar',
4829             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4830             'view_count': int,
4831             'description': '',
4832             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4833         },
4834         'playlist_mincount': 1123,
4835         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4836     }, {
4837         'note': 'even larger playlist, 8832 videos',
4838         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4839         'only_matching': True,
4840     }, {
4841         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4842         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4843         'info_dict': {
4844             'title': 'Uploads from Interstellar Movie',
4845             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4846             'uploader': 'Interstellar Movie',
4847             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4848             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4849             'tags': [],
4850             'view_count': int,
4851             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4852             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4853             'channel': 'Interstellar Movie',
4854             'description': '',
4855             'modified_date': r're:\d{8}',
4856         },
4857         'playlist_mincount': 21,
4858     }, {
4859         'note': 'Playlist with "show unavailable videos" button',
4860         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4861         'info_dict': {
4862             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4863             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4864             'uploader': 'Phim Siêu Nhân Nhật Bản',
4865             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4866             'view_count': int,
4867             'channel': 'Phim Siêu Nhân Nhật Bản',
4868             'tags': [],
4869             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4870             'description': '',
4871             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4872             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4873             'modified_date': r're:\d{8}',
4874         },
4875         'playlist_mincount': 200,
4876         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4877     }, {
4878         'note': 'Playlist with unavailable videos in page 7',
4879         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4880         'info_dict': {
4881             'title': 'Uploads from BlankTV',
4882             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4883             'uploader': 'BlankTV',
4884             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4885             'channel': 'BlankTV',
4886             'channel_url': 'https://www.youtube.com/c/blanktv',
4887             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4888             'view_count': int,
4889             'tags': [],
4890             'uploader_url': 'https://www.youtube.com/c/blanktv',
4891             'modified_date': r're:\d{8}',
4892             'description': '',
4893         },
4894         'playlist_mincount': 1000,
4895         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4896     }, {
4897         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4898         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4899         'info_dict': {
4900             'title': 'Data Analysis with Dr Mike Pound',
4901             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4902             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4903             'uploader': 'Computerphile',
4904             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4905             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4906             'tags': [],
4907             'view_count': int,
4908             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4909             'channel_url': 'https://www.youtube.com/user/Computerphile',
4910             'channel': 'Computerphile',
4911         },
4912         'playlist_mincount': 11,
4913     }, {
4914         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4915         'only_matching': True,
4916     }, {
4917         'note': 'Playlist URL that does not actually serve a playlist',
4918         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4919         'info_dict': {
4920             'id': 'FqZTN594JQw',
4921             'ext': 'webm',
4922             'title': "Smiley's People 01 detective, Adventure Series, Action",
4923             'uploader': 'STREEM',
4924             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4925             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4926             'upload_date': '20150526',
4927             'license': 'Standard YouTube License',
4928             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4929             'categories': ['People & Blogs'],
4930             'tags': list,
4931             'view_count': int,
4932             'like_count': int,
4933         },
4934         'params': {
4935             'skip_download': True,
4936         },
4937         'skip': 'This video is not available.',
4938         'add_ie': [YoutubeIE.ie_key()],
4939     }, {
4940         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
4941         'only_matching': True,
4942     }, {
4943         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
4944         'only_matching': True,
4945     }, {
4946         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
4947         'info_dict': {
4948             'id': 'GgL890LIznQ',  # This will keep changing
4949             'ext': 'mp4',
4950             'title': str,
4951             'uploader': 'Sky News',
4952             'uploader_id': 'skynews',
4953             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
4954             'upload_date': r're:\d{8}',
4955             'description': str,
4956             'categories': ['News & Politics'],
4957             'tags': list,
4958             'like_count': int,
4959             'release_timestamp': 1642502819,
4960             'channel': 'Sky News',
4961             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
4962             'age_limit': 0,
4963             'view_count': int,
4964             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
4965             'playable_in_embed': True,
4966             'release_date': '20220118',
4967             'availability': 'public',
4968             'live_status': 'is_live',
4969             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
4970             'channel_follower_count': int
4971         },
4972         'params': {
4973             'skip_download': True,
4974         },
4975         'expected_warnings': ['Ignoring subtitle tracks found in '],
4976     }, {
4977         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
4978         'info_dict': {
4979             'id': 'a48o2S1cPoo',
4980             'ext': 'mp4',
4981             'title': 'The Young Turks - Live Main Show',
4982             'uploader': 'The Young Turks',
4983             'uploader_id': 'TheYoungTurks',
4984             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
4985             'upload_date': '20150715',
4986             'license': 'Standard YouTube License',
4987             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
4988             'categories': ['News & Politics'],
4989             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
4990             'like_count': int,
4991         },
4992         'params': {
4993             'skip_download': True,
4994         },
4995         'only_matching': True,
4996     }, {
4997         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
4998         'only_matching': True,
4999     }, {
5000         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5001         'only_matching': True,
5002     }, {
5003         'note': 'A channel that is not live. Should raise error',
5004         'url': 'https://www.youtube.com/user/numberphile/live',
5005         'only_matching': True,
5006     }, {
5007         'url': 'https://www.youtube.com/feed/trending',
5008         'only_matching': True,
5009     }, {
5010         'url': 'https://www.youtube.com/feed/library',
5011         'only_matching': True,
5012     }, {
5013         'url': 'https://www.youtube.com/feed/history',
5014         'only_matching': True,
5015     }, {
5016         'url': 'https://www.youtube.com/feed/subscriptions',
5017         'only_matching': True,
5018     }, {
5019         'url': 'https://www.youtube.com/feed/watch_later',
5020         'only_matching': True,
5021     }, {
5022         'note': 'Recommended - redirects to home page.',
5023         'url': 'https://www.youtube.com/feed/recommended',
5024         'only_matching': True,
5025     }, {
5026         'note': 'inline playlist with not always working continuations',
5027         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5028         'only_matching': True,
5029     }, {
5030         'url': 'https://www.youtube.com/course',
5031         'only_matching': True,
5032     }, {
5033         'url': 'https://www.youtube.com/zsecurity',
5034         'only_matching': True,
5035     }, {
5036         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5037         'only_matching': True,
5038     }, {
5039         'url': 'https://www.youtube.com/TheYoungTurks/live',
5040         'only_matching': True,
5041     }, {
5042         'url': 'https://www.youtube.com/hashtag/cctv9',
5043         'info_dict': {
5044             'id': 'cctv9',
5045             'title': '#cctv9',
5046             'tags': [],
5047         },
5048         'playlist_mincount': 350,
5049     }, {
5050         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5051         'only_matching': True,
5052     }, {
5053         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5054         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5055         'only_matching': True
5056     }, {
5057         'note': '/browse/ should redirect to /channel/',
5058         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5059         'only_matching': True
5060     }, {
5061         'note': 'VLPL, should redirect to playlist?list=PL...',
5062         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5063         'info_dict': {
5064             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5065             'uploader': 'NoCopyrightSounds',
5066             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5067             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5068             'title': 'NCS Releases',
5069             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5070             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5071             'modified_date': r're:\d{8}',
5072             'view_count': int,
5073             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5074             'tags': [],
5075             'channel': 'NoCopyrightSounds',
5076         },
5077         'playlist_mincount': 166,
5078         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5079     }, {
5080         'note': 'Topic, should redirect to playlist?list=UU...',
5081         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5082         'info_dict': {
5083             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5084             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5085             'title': 'Uploads from Royalty Free Music - Topic',
5086             'uploader': 'Royalty Free Music - Topic',
5087             'tags': [],
5088             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5089             'channel': 'Royalty Free Music - Topic',
5090             'view_count': int,
5091             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5092             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5093             'modified_date': r're:\d{8}',
5094             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5095             'description': '',
5096         },
5097         'expected_warnings': [
5098             'The URL does not have a videos tab',
5099             r'[Uu]navailable videos (are|will be) hidden',
5100         ],
5101         'playlist_mincount': 101,
5102     }, {
5103         'note': 'Topic without a UU playlist',
5104         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5105         'info_dict': {
5106             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5107             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5108             'tags': [],
5109         },
5110         'expected_warnings': [
5111             'the playlist redirect gave error',
5112         ],
5113         'playlist_mincount': 9,
5114     }, {
5115         'note': 'Youtube music Album',
5116         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5117         'info_dict': {
5118             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5119             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5120             'tags': [],
5121             'view_count': int,
5122             'description': '',
5123             'availability': 'unlisted',
5124             'modified_date': r're:\d{8}',
5125         },
5126         'playlist_count': 50,
5127     }, {
5128         'note': 'unlisted single video playlist',
5129         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5130         'info_dict': {
5131             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5132             'uploader': 'colethedj',
5133             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5134             'title': 'yt-dlp unlisted playlist test',
5135             'availability': 'unlisted',
5136             'tags': [],
5137             'modified_date': '20211208',
5138             'channel': 'colethedj',
5139             'view_count': int,
5140             'description': '',
5141             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5142             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5143             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5144         },
5145         'playlist_count': 1,
5146     }, {
5147         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5148         'url': 'https://www.youtube.com/feed/recommended',
5149         'info_dict': {
5150             'id': 'recommended',
5151             'title': 'recommended',
5152             'tags': [],
5153         },
5154         'playlist_mincount': 50,
5155         'params': {
5156             'skip_download': True,
5157             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5158         },
5159     }, {
5160         'note': 'API Fallback: /videos tab, sorted by oldest first',
5161         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5162         'info_dict': {
5163             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5164             'title': 'Cody\'sLab - Videos',
5165             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5166             'uploader': 'Cody\'sLab',
5167             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5168             'channel': 'Cody\'sLab',
5169             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5170             'tags': [],
5171             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5172             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5173             'channel_follower_count': int
5174         },
5175         'playlist_mincount': 650,
5176         'params': {
5177             'skip_download': True,
5178             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5179         },
5180     }, {
5181         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5182         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5183         'info_dict': {
5184             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5185             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5186             'title': 'Uploads from Royalty Free Music - Topic',
5187             'uploader': 'Royalty Free Music - Topic',
5188             'modified_date': r're:\d{8}',
5189             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5190             'description': '',
5191             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5192             'tags': [],
5193             'channel': 'Royalty Free Music - Topic',
5194             'view_count': int,
5195             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5196         },
5197         'expected_warnings': [
5198             'does not have a videos tab',
5199             r'[Uu]navailable videos (are|will be) hidden',
5200         ],
5201         'playlist_mincount': 101,
5202         'params': {
5203             'skip_download': True,
5204             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5205         },
5206     }, {
5207         'note': 'non-standard redirect to regional channel',
5208         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5209         'only_matching': True
5210     }, {
5211         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5212         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5213         'info_dict': {
5214             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5215             'modified_date': '20220407',
5216             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5217             'tags': [],
5218             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5219             'uploader': 'pukkandan',
5220             'availability': 'unlisted',
5221             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5222             'channel': 'pukkandan',
5223             'description': 'Test for collaborative playlist',
5224             'title': 'yt-dlp test - collaborative playlist',
5225             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5226         },
5227         'playlist_mincount': 2
5228     }]
5229
5230     @classmethod
5231     def suitable(cls, url):
5232         return False if YoutubeIE.suitable(url) else super().suitable(url)
5233
5234     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5235
5236     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5237     def _real_extract(self, url, smuggled_data):
5238         item_id = self._match_id(url)
5239         url = compat_urlparse.urlunparse(
5240             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
5241         compat_opts = self.get_param('compat_opts', [])
5242
5243         def get_mobj(url):
5244             mobj = self._URL_RE.match(url).groupdict()
5245             mobj.update((k, '') for k, v in mobj.items() if v is None)
5246             return mobj
5247
5248         mobj, redirect_warning = get_mobj(url), None
5249         # Youtube returns incomplete data if tabname is not lower case
5250         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5251         if is_channel:
5252             if smuggled_data.get('is_music_url'):
5253                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5254                     item_id = item_id[2:]
5255                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5256                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5257                     mdata = self._extract_tab_endpoint(
5258                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5259                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5260                                         get_all=False, expected_type=compat_str)
5261                     if not murl:
5262                         raise ExtractorError('Failed to resolve album to playlist')
5263                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5264                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5265                     pre = f'https://www.youtube.com/channel/{item_id}'
5266
5267         original_tab_name = tab
5268         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5269             # Home URLs should redirect to /videos/
5270             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5271                                 'To download only the videos in the home page, add a "/featured" to the URL')
5272             tab = '/videos'
5273
5274         url = ''.join((pre, tab, post))
5275         mobj = get_mobj(url)
5276
5277         # Handle both video/playlist URLs
5278         qs = parse_qs(url)
5279         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5280
5281         if not video_id and mobj['not_channel'].startswith('watch'):
5282             if not playlist_id:
5283                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5284                 raise ExtractorError('Unable to recognize tab page')
5285             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5286             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5287             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5288             mobj = get_mobj(url)
5289
5290         if video_id and playlist_id:
5291             if self.get_param('noplaylist'):
5292                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5293                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5294                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5295             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5296
5297         data, ytcfg = self._extract_data(url, item_id)
5298
5299         # YouTube may provide a non-standard redirect to the regional channel
5300         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5301         redirect_url = traverse_obj(
5302             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5303         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5304             redirect_url = ''.join((
5305                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5306             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5307             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5308
5309         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5310         if tabs:
5311             selected_tab = self._extract_selected_tab(tabs)
5312             selected_tab_name = selected_tab.get('title', '').lower()
5313             if selected_tab_name == 'home':
5314                 selected_tab_name = 'featured'
5315             requested_tab_name = mobj['tab'][1:]
5316             if 'no-youtube-channel-redirect' not in compat_opts:
5317                 if requested_tab_name == 'live':
5318                     # Live tab should have redirected to the video
5319                     raise ExtractorError('The channel is not currently live', expected=True)
5320                 if requested_tab_name not in ('', selected_tab_name):
5321                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5322                     if not original_tab_name:
5323                         if item_id[:2] == 'UC':
5324                             # Topic channels don't have /videos. Use the equivalent playlist instead
5325                             pl_id = f'UU{item_id[2:]}'
5326                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5327                             try:
5328                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5329                             except ExtractorError:
5330                                 redirect_warning += ' and the playlist redirect gave error'
5331                             else:
5332                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5333                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5334                         if selected_tab_name and selected_tab_name != requested_tab_name:
5335                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5336                     else:
5337                         raise ExtractorError(redirect_warning, expected=True)
5338
5339         if redirect_warning:
5340             self.to_screen(redirect_warning)
5341         self.write_debug(f'Final URL: {url}')
5342
5343         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5344         if 'no-youtube-unavailable-videos' not in compat_opts:
5345             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5346         self._extract_and_report_alerts(data, only_once=True)
5347         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5348         if tabs:
5349             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5350
5351         playlist = traverse_obj(
5352             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5353         if playlist:
5354             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5355
5356         video_id = traverse_obj(
5357             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5358         if video_id:
5359             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5360                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5361             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5362                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5363
5364         raise ExtractorError('Unable to recognize tab page')
5365
5366
5367 class YoutubePlaylistIE(InfoExtractor):
5368     IE_DESC = 'YouTube playlists'
5369     _VALID_URL = r'''(?x)(?:
5370                         (?:https?://)?
5371                         (?:\w+\.)?
5372                         (?:
5373                             (?:
5374                                 youtube(?:kids)?\.com|
5375                                 %(invidious)s
5376                             )
5377                             /.*?\?.*?\blist=
5378                         )?
5379                         (?P<id>%(playlist_id)s)
5380                      )''' % {
5381         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5382         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5383     }
5384     IE_NAME = 'youtube:playlist'
5385     _TESTS = [{
5386         'note': 'issue #673',
5387         'url': 'PLBB231211A4F62143',
5388         'info_dict': {
5389             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5390             'id': 'PLBB231211A4F62143',
5391             'uploader': 'Wickman',
5392             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5393             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5394             'view_count': int,
5395             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5396             'modified_date': r're:\d{8}',
5397             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5398             'channel': 'Wickman',
5399             'tags': [],
5400             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5401         },
5402         'playlist_mincount': 29,
5403     }, {
5404         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5405         'info_dict': {
5406             'title': 'YDL_safe_search',
5407             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5408         },
5409         'playlist_count': 2,
5410         'skip': 'This playlist is private',
5411     }, {
5412         'note': 'embedded',
5413         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5414         'playlist_count': 4,
5415         'info_dict': {
5416             'title': 'JODA15',
5417             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5418             'uploader': 'milan',
5419             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5420             'description': '',
5421             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5422             'tags': [],
5423             'modified_date': '20140919',
5424             'view_count': int,
5425             'channel': 'milan',
5426             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5427             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5428         },
5429         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5430     }, {
5431         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5432         'playlist_mincount': 654,
5433         'info_dict': {
5434             'title': '2018 Chinese New Singles (11/6 updated)',
5435             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5436             'uploader': 'LBK',
5437             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5438             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5439             'channel': 'LBK',
5440             'view_count': int,
5441             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5442             'tags': [],
5443             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5444             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5445             'modified_date': r're:\d{8}',
5446         },
5447         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5448     }, {
5449         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5450         'only_matching': True,
5451     }, {
5452         # music album playlist
5453         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5454         'only_matching': True,
5455     }]
5456
5457     @classmethod
5458     def suitable(cls, url):
5459         if YoutubeTabIE.suitable(url):
5460             return False
5461         from ..utils import parse_qs
5462         qs = parse_qs(url)
5463         if qs.get('v', [None])[0]:
5464             return False
5465         return super().suitable(url)
5466
5467     def _real_extract(self, url):
5468         playlist_id = self._match_id(url)
5469         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5470         url = update_url_query(
5471             'https://www.youtube.com/playlist',
5472             parse_qs(url) or {'list': playlist_id})
5473         if is_music_url:
5474             url = smuggle_url(url, {'is_music_url': True})
5475         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5476
5477
5478 class YoutubeYtBeIE(InfoExtractor):
5479     IE_DESC = 'youtu.be'
5480     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5481     _TESTS = [{
5482         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5483         'info_dict': {
5484             'id': 'yeWKywCrFtk',
5485             'ext': 'mp4',
5486             'title': 'Small Scale Baler and Braiding Rugs',
5487             'uploader': 'Backus-Page House Museum',
5488             'uploader_id': 'backuspagemuseum',
5489             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5490             'upload_date': '20161008',
5491             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5492             'categories': ['Nonprofits & Activism'],
5493             'tags': list,
5494             'like_count': int,
5495             'age_limit': 0,
5496             'playable_in_embed': True,
5497             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5498             'channel': 'Backus-Page House Museum',
5499             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5500             'live_status': 'not_live',
5501             'view_count': int,
5502             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5503             'availability': 'public',
5504             'duration': 59,
5505         },
5506         'params': {
5507             'noplaylist': True,
5508             'skip_download': True,
5509         },
5510     }, {
5511         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5512         'only_matching': True,
5513     }]
5514
5515     def _real_extract(self, url):
5516         mobj = self._match_valid_url(url)
5517         video_id = mobj.group('id')
5518         playlist_id = mobj.group('playlist_id')
5519         return self.url_result(
5520             update_url_query('https://www.youtube.com/watch', {
5521                 'v': video_id,
5522                 'list': playlist_id,
5523                 'feature': 'youtu.be',
5524             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5525
5526
5527 class YoutubeLivestreamEmbedIE(InfoExtractor):
5528     IE_DESC = 'YouTube livestream embeds'
5529     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5530     _TESTS = [{
5531         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5532         'only_matching': True,
5533     }]
5534
5535     def _real_extract(self, url):
5536         channel_id = self._match_id(url)
5537         return self.url_result(
5538             f'https://www.youtube.com/channel/{channel_id}/live',
5539             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5540
5541
5542 class YoutubeYtUserIE(InfoExtractor):
5543     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5544     IE_NAME = 'youtube:user'
5545     _VALID_URL = r'ytuser:(?P<id>.+)'
5546     _TESTS = [{
5547         'url': 'ytuser:phihag',
5548         'only_matching': True,
5549     }]
5550
5551     def _real_extract(self, url):
5552         user_id = self._match_id(url)
5553         return self.url_result(
5554             'https://www.youtube.com/user/%s/videos' % user_id,
5555             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5556
5557
5558 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5559     IE_NAME = 'youtube:favorites'
5560     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5561     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5562     _LOGIN_REQUIRED = True
5563     _TESTS = [{
5564         'url': ':ytfav',
5565         'only_matching': True,
5566     }, {
5567         'url': ':ytfavorites',
5568         'only_matching': True,
5569     }]
5570
5571     def _real_extract(self, url):
5572         return self.url_result(
5573             'https://www.youtube.com/playlist?list=LL',
5574             ie=YoutubeTabIE.ie_key())
5575
5576
5577 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5578     IE_NAME = 'youtube:notif'
5579     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5580     _VALID_URL = r':ytnotif(?:ication)?s?'
5581     _LOGIN_REQUIRED = True
5582     _TESTS = [{
5583         'url': ':ytnotif',
5584         'only_matching': True,
5585     }, {
5586         'url': ':ytnotifications',
5587         'only_matching': True,
5588     }]
5589
5590     def _extract_notification_menu(self, response, continuation_list):
5591         notification_list = traverse_obj(
5592             response,
5593             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5594             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5595             expected_type=list) or []
5596         continuation_list[0] = None
5597         for item in notification_list:
5598             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5599             if entry:
5600                 yield entry
5601             continuation = item.get('continuationItemRenderer')
5602             if continuation:
5603                 continuation_list[0] = continuation
5604
5605     def _extract_notification_renderer(self, notification):
5606         video_id = traverse_obj(
5607             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5608         url = f'https://www.youtube.com/watch?v={video_id}'
5609         channel_id = None
5610         if not video_id:
5611             browse_ep = traverse_obj(
5612                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5613             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5614             post_id = self._search_regex(
5615                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5616                 'post id', default=None)
5617             if not channel_id or not post_id:
5618                 return
5619             # The direct /post url redirects to this in the browser
5620             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5621
5622         channel = traverse_obj(
5623             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5624             expected_type=str)
5625         title = self._search_regex(
5626             rf'{re.escape(channel)} [^:]+: (.+)', self._get_text(notification, 'shortMessage'),
5627             'video title', default=None)
5628         if title:
5629             title = title.replace('\xad', '')  # remove soft hyphens
5630         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5631                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5632                        else None)
5633         return {
5634             '_type': 'url',
5635             'url': url,
5636             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5637             'video_id': video_id,
5638             'title': title,
5639             'channel_id': channel_id,
5640             'channel': channel,
5641             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5642             'upload_date': upload_date,
5643         }
5644
5645     def _notification_menu_entries(self, ytcfg):
5646         continuation_list = [None]
5647         response = None
5648         for page in itertools.count(1):
5649             ctoken = traverse_obj(
5650                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5651             response = self._extract_response(
5652                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5653                 ep='notification/get_notification_menu', check_get_keys='actions',
5654                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5655             yield from self._extract_notification_menu(response, continuation_list)
5656             if not continuation_list[0]:
5657                 break
5658
5659     def _real_extract(self, url):
5660         display_id = 'notifications'
5661         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5662         self._report_playlist_authcheck(ytcfg)
5663         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5664
5665
5666 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5667     IE_DESC = 'YouTube search'
5668     IE_NAME = 'youtube:search'
5669     _SEARCH_KEY = 'ytsearch'
5670     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5671     _TESTS = [{
5672         'url': 'ytsearch5:youtube-dl test video',
5673         'playlist_count': 5,
5674         'info_dict': {
5675             'id': 'youtube-dl test video',
5676             'title': 'youtube-dl test video',
5677         }
5678     }]
5679
5680
5681 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5682     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5683     _SEARCH_KEY = 'ytsearchdate'
5684     IE_DESC = 'YouTube search, newest videos first'
5685     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5686     _TESTS = [{
5687         'url': 'ytsearchdate5:youtube-dl test video',
5688         'playlist_count': 5,
5689         'info_dict': {
5690             'id': 'youtube-dl test video',
5691             'title': 'youtube-dl test video',
5692         }
5693     }]
5694
5695
5696 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5697     IE_DESC = 'YouTube search URLs with sorting and filter support'
5698     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5699     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5700     _TESTS = [{
5701         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5702         'playlist_mincount': 5,
5703         'info_dict': {
5704             'id': 'youtube-dl test video',
5705             'title': 'youtube-dl test video',
5706         }
5707     }, {
5708         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5709         'playlist_mincount': 5,
5710         'info_dict': {
5711             'id': 'python',
5712             'title': 'python',
5713         }
5714     }, {
5715         'url': 'https://www.youtube.com/results?search_query=%23cats',
5716         'playlist_mincount': 1,
5717         'info_dict': {
5718             'id': '#cats',
5719             'title': '#cats',
5720             'entries': [{
5721                 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5722                 'title': '#cats',
5723             }],
5724         },
5725     }, {
5726         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5727         'only_matching': True,
5728     }]
5729
5730     def _real_extract(self, url):
5731         qs = parse_qs(url)
5732         query = (qs.get('search_query') or qs.get('q'))[0]
5733         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5734
5735
5736 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5737     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5738     IE_NAME = 'youtube:music:search_url'
5739     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5740     _TESTS = [{
5741         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5742         'playlist_count': 16,
5743         'info_dict': {
5744             'id': 'royalty free music',
5745             'title': 'royalty free music',
5746         }
5747     }, {
5748         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5749         'playlist_mincount': 30,
5750         'info_dict': {
5751             'id': 'royalty free music - songs',
5752             'title': 'royalty free music - songs',
5753         },
5754         'params': {'extract_flat': 'in_playlist'}
5755     }, {
5756         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5757         'playlist_mincount': 30,
5758         'info_dict': {
5759             'id': 'royalty free music - community playlists',
5760             'title': 'royalty free music - community playlists',
5761         },
5762         'params': {'extract_flat': 'in_playlist'}
5763     }]
5764
5765     _SECTIONS = {
5766         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5767         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5768         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5769         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5770         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5771         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5772     }
5773
5774     def _real_extract(self, url):
5775         qs = parse_qs(url)
5776         query = (qs.get('search_query') or qs.get('q'))[0]
5777         params = qs.get('sp', (None,))[0]
5778         if params:
5779             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5780         else:
5781             section = compat_urllib_parse_unquote_plus((url.split('#') + [''])[1]).lower()
5782             params = self._SECTIONS.get(section)
5783             if not params:
5784                 section = None
5785         title = join_nonempty(query, section, delim=' - ')
5786         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5787
5788
5789 class YoutubeFeedsInfoExtractor(InfoExtractor):
5790     """
5791     Base class for feed extractors
5792     Subclasses must re-define the _FEED_NAME property.
5793     """
5794     _LOGIN_REQUIRED = True
5795     _FEED_NAME = 'feeds'
5796
5797     def _real_initialize(self):
5798         YoutubeBaseInfoExtractor._check_login_required(self)
5799
5800     @classproperty
5801     def IE_NAME(self):
5802         return f'youtube:{self._FEED_NAME}'
5803
5804     def _real_extract(self, url):
5805         return self.url_result(
5806             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5807
5808
5809 class YoutubeWatchLaterIE(InfoExtractor):
5810     IE_NAME = 'youtube:watchlater'
5811     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5812     _VALID_URL = r':ytwatchlater'
5813     _TESTS = [{
5814         'url': ':ytwatchlater',
5815         'only_matching': True,
5816     }]
5817
5818     def _real_extract(self, url):
5819         return self.url_result(
5820             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5821
5822
5823 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5824     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5825     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5826     _FEED_NAME = 'recommended'
5827     _LOGIN_REQUIRED = False
5828     _TESTS = [{
5829         'url': ':ytrec',
5830         'only_matching': True,
5831     }, {
5832         'url': ':ytrecommended',
5833         'only_matching': True,
5834     }, {
5835         'url': 'https://youtube.com',
5836         'only_matching': True,
5837     }]
5838
5839
5840 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5841     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5842     _VALID_URL = r':ytsub(?:scription)?s?'
5843     _FEED_NAME = 'subscriptions'
5844     _TESTS = [{
5845         'url': ':ytsubs',
5846         'only_matching': True,
5847     }, {
5848         'url': ':ytsubscriptions',
5849         'only_matching': True,
5850     }]
5851
5852
5853 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5854     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5855     _VALID_URL = r':ythis(?:tory)?'
5856     _FEED_NAME = 'history'
5857     _TESTS = [{
5858         'url': ':ythistory',
5859         'only_matching': True,
5860     }]
5861
5862
5863 class YoutubeStoriesIE(InfoExtractor):
5864     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5865     IE_NAME = 'youtube:stories'
5866     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5867     _TESTS = [{
5868         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5869         'only_matching': True,
5870     }]
5871
5872     def _real_extract(self, url):
5873         playlist_id = f'RLTD{self._match_id(url)}'
5874         return self.url_result(
5875             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5876             ie=YoutubeTabIE, video_id=playlist_id)
5877
5878
5879 class YoutubeTruncatedURLIE(InfoExtractor):
5880     IE_NAME = 'youtube:truncated_url'
5881     IE_DESC = False  # Do not list
5882     _VALID_URL = r'''(?x)
5883         (?:https?://)?
5884         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5885         (?:watch\?(?:
5886             feature=[a-z_]+|
5887             annotation_id=annotation_[^&]+|
5888             x-yt-cl=[0-9]+|
5889             hl=[^&]*|
5890             t=[0-9]+
5891         )?
5892         |
5893             attribution_link\?a=[^&]+
5894         )
5895         $
5896     '''
5897
5898     _TESTS = [{
5899         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5900         'only_matching': True,
5901     }, {
5902         'url': 'https://www.youtube.com/watch?',
5903         'only_matching': True,
5904     }, {
5905         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5906         'only_matching': True,
5907     }, {
5908         'url': 'https://www.youtube.com/watch?feature=foo',
5909         'only_matching': True,
5910     }, {
5911         'url': 'https://www.youtube.com/watch?hl=en-GB',
5912         'only_matching': True,
5913     }, {
5914         'url': 'https://www.youtube.com/watch?t=2372',
5915         'only_matching': True,
5916     }]
5917
5918     def _real_extract(self, url):
5919         raise ExtractorError(
5920             'Did you forget to quote the URL? Remember that & is a meta '
5921             'character in most shells, so you want to put the URL in quotes, '
5922             'like  youtube-dl '
5923             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5924             ' or simply  youtube-dl BaW_jenozKc  .',
5925             expected=True)
5926
5927
5928 class YoutubeClipIE(InfoExtractor):
5929     IE_NAME = 'youtube:clip'
5930     IE_DESC = False  # Do not list
5931     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/'
5932
5933     def _real_extract(self, url):
5934         self.report_warning('YouTube clips are not currently supported. The entire video will be downloaded instead')
5935         return self.url_result(url, 'Generic')
5936
5937
5938 class YoutubeTruncatedIDIE(InfoExtractor):
5939     IE_NAME = 'youtube:truncated_id'
5940     IE_DESC = False  # Do not list
5941     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
5942
5943     _TESTS = [{
5944         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
5945         'only_matching': True,
5946     }]
5947
5948     def _real_extract(self, url):
5949         video_id = self._match_id(url)
5950         raise ExtractorError(
5951             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
5952             expected=True)