yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16
  17 from .common import InfoExtractor, SearchInfoExtractor
  18 from ..compat import functools  # isort: split
  19 from ..compat import (
  20     compat_chr,
  21     compat_HTTPError,
  22     compat_parse_qs,
  23     compat_str,
  24     compat_urllib_parse_unquote_plus,
  25     compat_urllib_parse_urlencode,
  26     compat_urllib_parse_urlparse,
  27     compat_urlparse,
  28 )
  29 from ..jsinterp import JSInterpreter
  30 from ..utils import (
  31     NO_DEFAULT,
  32     ExtractorError,
  33     bug_reports_message,
  34     classproperty,
  35     clean_html,
  36     datetime_from_str,
  37     dict_get,
  38     error_to_compat_str,
  39     float_or_none,
  40     format_field,
  41     get_first,
  42     int_or_none,
  43     is_html,
  44     join_nonempty,
  45     js_to_json,
  46     mimetype2ext,
  47     network_exceptions,
  48     orderedSet,
  49     parse_codecs,
  50     parse_count,
  51     parse_duration,
  52     parse_iso8601,
  53     parse_qs,
  54     qualities,
  55     remove_end,
  56     remove_start,
  57     smuggle_url,
  58     str_or_none,
  59     str_to_int,
  60     strftime_or_none,
  61     traverse_obj,
  62     try_get,
  63     unescapeHTML,
  64     unified_strdate,
  65     unified_timestamp,
  66     unsmuggle_url,
  67     update_url_query,
  68     url_or_none,
  69     urljoin,
  70     variadic,
  71 )
  72
  73 # any clients starting with _ cannot be explicity requested by the user
  74 INNERTUBE_CLIENTS = {
  75     'web': {
  76         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  77         'INNERTUBE_CONTEXT': {
  78             'client': {
  79                 'clientName': 'WEB',
  80                 'clientVersion': '2.20211221.00.00',
  81             }
  82         },
  83         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  84     },
  85     'web_embedded': {
  86         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  87         'INNERTUBE_CONTEXT': {
  88             'client': {
  89                 'clientName': 'WEB_EMBEDDED_PLAYER',
  90                 'clientVersion': '1.20211215.00.01',
  91             },
  92         },
  93         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  94     },
  95     'web_music': {
  96         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  97         'INNERTUBE_HOST': 'music.youtube.com',
  98         'INNERTUBE_CONTEXT': {
  99             'client': {
 100                 'clientName': 'WEB_REMIX',
 101                 'clientVersion': '1.20211213.00.00',
 102             }
 103         },
 104         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 105     },
 106     'web_creator': {
 107         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 108         'INNERTUBE_CONTEXT': {
 109             'client': {
 110                 'clientName': 'WEB_CREATOR',
 111                 'clientVersion': '1.20211220.02.00',
 112             }
 113         },
 114         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 115     },
 116     'android': {
 117         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 118         'INNERTUBE_CONTEXT': {
 119             'client': {
 120                 'clientName': 'ANDROID',
 121                 'clientVersion': '16.49',
 122             }
 123         },
 124         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 125         'REQUIRE_JS_PLAYER': False
 126     },
 127     'android_embedded': {
 128         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 129         'INNERTUBE_CONTEXT': {
 130             'client': {
 131                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 132                 'clientVersion': '16.49',
 133             },
 134         },
 135         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 136         'REQUIRE_JS_PLAYER': False
 137     },
 138     'android_music': {
 139         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 140         'INNERTUBE_CONTEXT': {
 141             'client': {
 142                 'clientName': 'ANDROID_MUSIC',
 143                 'clientVersion': '4.57',
 144             }
 145         },
 146         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 147         'REQUIRE_JS_PLAYER': False
 148     },
 149     'android_creator': {
 150         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 151         'INNERTUBE_CONTEXT': {
 152             'client': {
 153                 'clientName': 'ANDROID_CREATOR',
 154                 'clientVersion': '21.47',
 155             },
 156         },
 157         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 158         'REQUIRE_JS_PLAYER': False
 159     },
 160     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 161     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 162     'ios': {
 163         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 164         'INNERTUBE_CONTEXT': {
 165             'client': {
 166                 'clientName': 'IOS',
 167                 'clientVersion': '16.46',
 168                 'deviceModel': 'iPhone14,3',
 169             }
 170         },
 171         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 172         'REQUIRE_JS_PLAYER': False
 173     },
 174     'ios_embedded': {
 175         'INNERTUBE_CONTEXT': {
 176             'client': {
 177                 'clientName': 'IOS_MESSAGES_EXTENSION',
 178                 'clientVersion': '16.46',
 179                 'deviceModel': 'iPhone14,3',
 180             },
 181         },
 182         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 183         'REQUIRE_JS_PLAYER': False
 184     },
 185     'ios_music': {
 186         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 187         'INNERTUBE_CONTEXT': {
 188             'client': {
 189                 'clientName': 'IOS_MUSIC',
 190                 'clientVersion': '4.57',
 191             },
 192         },
 193         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 194         'REQUIRE_JS_PLAYER': False
 195     },
 196     'ios_creator': {
 197         'INNERTUBE_CONTEXT': {
 198             'client': {
 199                 'clientName': 'IOS_CREATOR',
 200                 'clientVersion': '21.47',
 201             },
 202         },
 203         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 204         'REQUIRE_JS_PLAYER': False
 205     },
 206     # mweb has 'ultralow' formats
 207     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 208     'mweb': {
 209         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 210         'INNERTUBE_CONTEXT': {
 211             'client': {
 212                 'clientName': 'MWEB',
 213                 'clientVersion': '2.20211221.01.00',
 214             }
 215         },
 216         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 217     },
 218     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 219     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 220     'tv_embedded': {
 221         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 222         'INNERTUBE_CONTEXT': {
 223             'client': {
 224                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 225                 'clientVersion': '2.0',
 226             },
 227         },
 228         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 229     },
 230 }
 231
 232
 233 def _split_innertube_client(client_name):
 234     variant, *base = client_name.rsplit('.', 1)
 235     if base:
 236         return variant, base[0], variant
 237     base, *variant = client_name.split('_', 1)
 238     return client_name, base, variant[0] if variant else None
 239
 240
 241 def build_innertube_clients():
 242     THIRD_PARTY = {
 243         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 244     }
 245     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 246     priority = qualities(BASE_CLIENTS[::-1])
 247
 248     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 249         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 250         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 251         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 252         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 253
 254         _, base_client, variant = _split_innertube_client(client)
 255         ytcfg['priority'] = 10 * priority(base_client)
 256
 257         if not variant:
 258             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 259             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 260             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 261             embedscreen['priority'] -= 3
 262         elif variant == 'embedded':
 263             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 264             ytcfg['priority'] -= 2
 265         else:
 266             ytcfg['priority'] -= 3
 267
 268
 269 build_innertube_clients()
 270
 271
 272 class YoutubeBaseInfoExtractor(InfoExtractor):
 273     """Provide base functions for Youtube extractors"""
 274
 275     _RESERVED_NAMES = (
 276         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 277         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 278         r'browse|oembed|get_video_info|iframe_api|s/player|'
 279         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 280
 281     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 282
 283     # _NETRC_MACHINE = 'youtube'
 284
 285     # If True it will raise an error if no login info is provided
 286     _LOGIN_REQUIRED = False
 287
 288     _INVIDIOUS_SITES = (
 289         # invidious-redirect websites
 290         r'(?:www\.)?redirect\.invidious\.io',
 291         r'(?:(?:www|dev)\.)?invidio\.us',
 292         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 293         r'(?:www\.)?invidious\.pussthecat\.org',
 294         r'(?:www\.)?invidious\.zee\.li',
 295         r'(?:www\.)?invidious\.ethibox\.fr',
 296         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 297         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 298         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 299         # youtube-dl invidious instances list
 300         r'(?:(?:www|no)\.)?invidiou\.sh',
 301         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 302         r'(?:www\.)?invidious\.kabi\.tk',
 303         r'(?:www\.)?invidious\.mastodon\.host',
 304         r'(?:www\.)?invidious\.zapashcanon\.fr',
 305         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 306         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 307         r'(?:www\.)?invidious\.himiko\.cloud',
 308         r'(?:www\.)?invidious\.reallyancient\.tech',
 309         r'(?:www\.)?invidious\.tube',
 310         r'(?:www\.)?invidiou\.site',
 311         r'(?:www\.)?invidious\.site',
 312         r'(?:www\.)?invidious\.xyz',
 313         r'(?:www\.)?invidious\.nixnet\.xyz',
 314         r'(?:www\.)?invidious\.048596\.xyz',
 315         r'(?:www\.)?invidious\.drycat\.fr',
 316         r'(?:www\.)?inv\.skyn3t\.in',
 317         r'(?:www\.)?tube\.poal\.co',
 318         r'(?:www\.)?tube\.connect\.cafe',
 319         r'(?:www\.)?vid\.wxzm\.sx',
 320         r'(?:www\.)?vid\.mint\.lgbt',
 321         r'(?:www\.)?vid\.puffyan\.us',
 322         r'(?:www\.)?yewtu\.be',
 323         r'(?:www\.)?yt\.elukerio\.org',
 324         r'(?:www\.)?yt\.lelux\.fi',
 325         r'(?:www\.)?invidious\.ggc-project\.de',
 326         r'(?:www\.)?yt\.maisputain\.ovh',
 327         r'(?:www\.)?ytprivate\.com',
 328         r'(?:www\.)?invidious\.13ad\.de',
 329         r'(?:www\.)?invidious\.toot\.koeln',
 330         r'(?:www\.)?invidious\.fdn\.fr',
 331         r'(?:www\.)?watch\.nettohikari\.com',
 332         r'(?:www\.)?invidious\.namazso\.eu',
 333         r'(?:www\.)?invidious\.silkky\.cloud',
 334         r'(?:www\.)?invidious\.exonip\.de',
 335         r'(?:www\.)?invidious\.riverside\.rocks',
 336         r'(?:www\.)?invidious\.blamefran\.net',
 337         r'(?:www\.)?invidious\.moomoo\.de',
 338         r'(?:www\.)?ytb\.trom\.tf',
 339         r'(?:www\.)?yt\.cyberhost\.uk',
 340         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 341         r'(?:www\.)?qklhadlycap4cnod\.onion',
 342         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 343         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 344         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 345         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 346         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 347         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 348         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 349         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 350         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 351         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 352         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 353         r'(?:www\.)?piped\.kavin\.rocks',
 354         r'(?:www\.)?piped\.silkky\.cloud',
 355         r'(?:www\.)?piped\.tokhmi\.xyz',
 356         r'(?:www\.)?piped\.moomoo\.me',
 357         r'(?:www\.)?il\.ax',
 358         r'(?:www\.)?piped\.syncpundit\.com',
 359         r'(?:www\.)?piped\.mha\.fi',
 360         r'(?:www\.)?piped\.mint\.lgbt',
 361         r'(?:www\.)?piped\.privacy\.com\.de',
 362     )
 363
 364     def _initialize_consent(self):
 365         cookies = self._get_cookies('https://www.youtube.com/')
 366         if cookies.get('__Secure-3PSID'):
 367             return
 368         consent_id = None
 369         consent = cookies.get('CONSENT')
 370         if consent:
 371             if 'YES' in consent.value:
 372                 return
 373             consent_id = self._search_regex(
 374                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 375         if not consent_id:
 376             consent_id = random.randint(100, 999)
 377         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 378
 379     def _initialize_pref(self):
 380         cookies = self._get_cookies('https://www.youtube.com/')
 381         pref_cookie = cookies.get('PREF')
 382         pref = {}
 383         if pref_cookie:
 384             try:
 385                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 386             except ValueError:
 387                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 388         pref.update({'hl': 'en', 'tz': 'UTC'})
 389         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 390
 391     def _real_initialize(self):
 392         self._initialize_pref()
 393         self._initialize_consent()
 394         self._check_login_required()
 395
 396     def _check_login_required(self):
 397         if self._LOGIN_REQUIRED and not self._cookies_passed:
 398             self.raise_login_required('Login details are needed to download this content', method='cookies')
 399
 400     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 401     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 402
 403     def _get_default_ytcfg(self, client='web'):
 404         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 405
 406     def _get_innertube_host(self, client='web'):
 407         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 408
 409     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 410         # try_get but with fallback to default ytcfg client values when present
 411         _func = lambda y: try_get(y, getter, expected_type)
 412         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 413
 414     def _extract_client_name(self, ytcfg, default_client='web'):
 415         return self._ytcfg_get_safe(
 416             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 417                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 418
 419     def _extract_client_version(self, ytcfg, default_client='web'):
 420         return self._ytcfg_get_safe(
 421             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 422                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 423
 424     def _select_api_hostname(self, req_api_hostname, default_client=None):
 425         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 426                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 427
 428     def _extract_api_key(self, ytcfg=None, default_client='web'):
 429         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 430
 431     def _extract_context(self, ytcfg=None, default_client='web'):
 432         context = get_first(
 433             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 434         # Enforce language and tz for extraction
 435         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 436         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 437         return context
 438
 439     _SAPISID = None
 440
 441     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 442         time_now = round(time.time())
 443         if self._SAPISID is None:
 444             yt_cookies = self._get_cookies('https://www.youtube.com')
 445             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 446             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 447             sapisid_cookie = dict_get(
 448                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 449             if sapisid_cookie and sapisid_cookie.value:
 450                 self._SAPISID = sapisid_cookie.value
 451                 self.write_debug('Extracted SAPISID cookie')
 452                 # SAPISID cookie is required if not already present
 453                 if not yt_cookies.get('SAPISID'):
 454                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 455                     self._set_cookie(
 456                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 457             else:
 458                 self._SAPISID = False
 459         if not self._SAPISID:
 460             return None
 461         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 462         sapisidhash = hashlib.sha1(
 463             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 464         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 465
 466     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 467                   note='Downloading API JSON', errnote='Unable to download API page',
 468                   context=None, api_key=None, api_hostname=None, default_client='web'):
 469
 470         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 471         data.update(query)
 472         real_headers = self.generate_api_headers(default_client=default_client)
 473         real_headers.update({'content-type': 'application/json'})
 474         if headers:
 475             real_headers.update(headers)
 476         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 477                    or api_key or self._extract_api_key(default_client=default_client))
 478         return self._download_json(
 479             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 480             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 481             data=json.dumps(data).encode('utf8'), headers=real_headers,
 482             query={'key': api_key, 'prettyPrint': 'false'})
 483
 484     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 485         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 486
 487     @staticmethod
 488     def _extract_session_index(*data):
 489         """
 490         Index of current account in account list.
 491         See: https://github.com/yt-dlp/yt-dlp/pull/519
 492         """
 493         for ytcfg in data:
 494             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 495             if session_index is not None:
 496                 return session_index
 497
 498     # Deprecated?
 499     def _extract_identity_token(self, ytcfg=None, webpage=None):
 500         if ytcfg:
 501             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 502             if token:
 503                 return token
 504         if webpage:
 505             return self._search_regex(
 506                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 507                 'identity token', default=None, fatal=False)
 508
 509     @staticmethod
 510     def _extract_account_syncid(*args):
 511         """
 512         Extract syncId required to download private playlists of secondary channels
 513         @params response and/or ytcfg
 514         """
 515         for data in args:
 516             # ytcfg includes channel_syncid if on secondary channel
 517             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 518             if delegated_sid:
 519                 return delegated_sid
 520             sync_ids = (try_get(
 521                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 522                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 523             if len(sync_ids) >= 2 and sync_ids[1]:
 524                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 525                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 526                 return sync_ids[0]
 527
 528     @staticmethod
 529     def _extract_visitor_data(*args):
 530         """
 531         Extracts visitorData from an API response or ytcfg
 532         Appears to be used to track session state
 533         """
 534         return get_first(
 535             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 536             expected_type=str)
 537
 538     @functools.cached_property
 539     def is_authenticated(self):
 540         return bool(self._generate_sapisidhash_header())
 541
 542     def extract_ytcfg(self, video_id, webpage):
 543         if not webpage:
 544             return {}
 545         return self._parse_json(
 546             self._search_regex(
 547                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 548                 default='{}'), video_id, fatal=False) or {}
 549
 550     def generate_api_headers(
 551             self, *, ytcfg=None, account_syncid=None, session_index=None,
 552             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 553
 554         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 555         headers = {
 556             'X-YouTube-Client-Name': compat_str(
 557                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 558             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 559             'Origin': origin,
 560             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 561             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 562             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 563         }
 564         if session_index is None:
 565             session_index = self._extract_session_index(ytcfg)
 566         if account_syncid or session_index is not None:
 567             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 568
 569         auth = self._generate_sapisidhash_header(origin)
 570         if auth is not None:
 571             headers['Authorization'] = auth
 572             headers['X-Origin'] = origin
 573         return {h: v for h, v in headers.items() if v is not None}
 574
 575     def _download_ytcfg(self, client, video_id):
 576         url = {
 577             'web': 'https://www.youtube.com',
 578             'web_music': 'https://music.youtube.com',
 579             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 580         }.get(client)
 581         if not url:
 582             return {}
 583         webpage = self._download_webpage(
 584             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 585         return self.extract_ytcfg(video_id, webpage) or {}
 586
 587     @staticmethod
 588     def _build_api_continuation_query(continuation, ctp=None):
 589         query = {
 590             'continuation': continuation
 591         }
 592         # TODO: Inconsistency with clickTrackingParams.
 593         # Currently we have a fixed ctp contained within context (from ytcfg)
 594         # and a ctp in root query for continuation.
 595         if ctp:
 596             query['clickTracking'] = {'clickTrackingParams': ctp}
 597         return query
 598
 599     @classmethod
 600     def _extract_next_continuation_data(cls, renderer):
 601         next_continuation = try_get(
 602             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 603                        lambda x: x['continuation']['reloadContinuationData']), dict)
 604         if not next_continuation:
 605             return
 606         continuation = next_continuation.get('continuation')
 607         if not continuation:
 608             return
 609         ctp = next_continuation.get('clickTrackingParams')
 610         return cls._build_api_continuation_query(continuation, ctp)
 611
 612     @classmethod
 613     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 614         if isinstance(continuation_ep, dict):
 615             continuation = try_get(
 616                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 617             if not continuation:
 618                 return
 619             ctp = continuation_ep.get('clickTrackingParams')
 620             return cls._build_api_continuation_query(continuation, ctp)
 621
 622     @classmethod
 623     def _extract_continuation(cls, renderer):
 624         next_continuation = cls._extract_next_continuation_data(renderer)
 625         if next_continuation:
 626             return next_continuation
 627
 628         contents = []
 629         for key in ('contents', 'items'):
 630             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 631
 632         for content in contents:
 633             if not isinstance(content, dict):
 634                 continue
 635             continuation_ep = try_get(
 636                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 637                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 638                 dict)
 639             continuation = cls._extract_continuation_ep_data(continuation_ep)
 640             if continuation:
 641                 return continuation
 642
 643     @classmethod
 644     def _extract_alerts(cls, data):
 645         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 646             if not isinstance(alert_dict, dict):
 647                 continue
 648             for alert in alert_dict.values():
 649                 alert_type = alert.get('type')
 650                 if not alert_type:
 651                     continue
 652                 message = cls._get_text(alert, 'text')
 653                 if message:
 654                     yield alert_type, message
 655
 656     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 657         errors = []
 658         warnings = []
 659         for alert_type, alert_message in alerts:
 660             if alert_type.lower() == 'error' and fatal:
 661                 errors.append([alert_type, alert_message])
 662             else:
 663                 warnings.append([alert_type, alert_message])
 664
 665         for alert_type, alert_message in (warnings + errors[:-1]):
 666             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 667         if errors:
 668             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 669
 670     def _extract_and_report_alerts(self, data, *args, **kwargs):
 671         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 672
 673     def _extract_badges(self, renderer: dict):
 674         badges = set()
 675         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 676             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 677             if label:
 678                 badges.add(label.lower())
 679         return badges
 680
 681     @staticmethod
 682     def _get_text(data, *path_list, max_runs=None):
 683         for path in path_list or [None]:
 684             if path is None:
 685                 obj = [data]
 686             else:
 687                 obj = traverse_obj(data, path, default=[])
 688                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 689                     obj = [obj]
 690             for item in obj:
 691                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 692                 if text:
 693                     return text
 694                 runs = try_get(item, lambda x: x['runs'], list) or []
 695                 if not runs and isinstance(item, list):
 696                     runs = item
 697
 698                 runs = runs[:min(len(runs), max_runs or len(runs))]
 699                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 700                 if text:
 701                     return text
 702
 703     def _get_count(self, data, *path_list):
 704         count_text = self._get_text(data, *path_list) or ''
 705         count = parse_count(count_text)
 706         if count is None:
 707             count = str_to_int(
 708                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 709         return count
 710
 711     @staticmethod
 712     def _extract_thumbnails(data, *path_list):
 713         """
 714         Extract thumbnails from thumbnails dict
 715         @param path_list: path list to level that contains 'thumbnails' key
 716         """
 717         thumbnails = []
 718         for path in path_list or [()]:
 719             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 720                 thumbnail_url = url_or_none(thumbnail.get('url'))
 721                 if not thumbnail_url:
 722                     continue
 723                 # Sometimes youtube gives a wrong thumbnail URL. See:
 724                 # https://github.com/yt-dlp/yt-dlp/issues/233
 725                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 726                 if 'maxresdefault' in thumbnail_url:
 727                     thumbnail_url = thumbnail_url.split('?')[0]
 728                 thumbnails.append({
 729                     'url': thumbnail_url,
 730                     'height': int_or_none(thumbnail.get('height')),
 731                     'width': int_or_none(thumbnail.get('width')),
 732                 })
 733         return thumbnails
 734
 735     @staticmethod
 736     def extract_relative_time(relative_time_text):
 737         """
 738         Extracts a relative time from string and converts to dt object
 739         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 740         """
 741         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 742         if mobj:
 743             start = mobj.group('start')
 744             if start:
 745                 return datetime_from_str(start)
 746             try:
 747                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 748             except ValueError:
 749                 return None
 750
 751     def _extract_time_text(self, renderer, *path_list):
 752         """@returns (timestamp, time_text)"""
 753         text = self._get_text(renderer, *path_list) or ''
 754         dt = self.extract_relative_time(text)
 755         timestamp = None
 756         if isinstance(dt, datetime.datetime):
 757             timestamp = calendar.timegm(dt.timetuple())
 758
 759         if timestamp is None:
 760             timestamp = (
 761                 unified_timestamp(text) or unified_timestamp(
 762                     self._search_regex(
 763                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 764                         text.lower(), 'time text', default=None)))
 765
 766         if text and timestamp is None:
 767             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 768         return timestamp, text
 769
 770     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 771                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 772                           default_client='web'):
 773         response = None
 774         last_error = None
 775         count = -1
 776         retries = self.get_param('extractor_retries', 3)
 777         if check_get_keys is None:
 778             check_get_keys = []
 779         while count < retries:
 780             count += 1
 781             if last_error:
 782                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 783             try:
 784                 response = self._call_api(
 785                     ep=ep, fatal=True, headers=headers,
 786                     video_id=item_id, query=query,
 787                     context=self._extract_context(ytcfg, default_client),
 788                     api_key=self._extract_api_key(ytcfg, default_client),
 789                     api_hostname=api_hostname, default_client=default_client,
 790                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 791             except ExtractorError as e:
 792                 if isinstance(e.cause, network_exceptions):
 793                     if isinstance(e.cause, compat_HTTPError):
 794                         first_bytes = e.cause.read(512)
 795                         if not is_html(first_bytes):
 796                             yt_error = try_get(
 797                                 self._parse_json(
 798                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 799                                 lambda x: x['error']['message'], compat_str)
 800                             if yt_error:
 801                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 802                     # Downloading page may result in intermittent 5xx HTTP error
 803                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 804                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 805                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 806                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 807                         last_error = error_to_compat_str(e.cause or e.msg)
 808                         if count < retries:
 809                             continue
 810                 if fatal:
 811                     raise
 812                 else:
 813                     self.report_warning(error_to_compat_str(e))
 814                     return
 815
 816             else:
 817                 try:
 818                     self._extract_and_report_alerts(response, only_once=True)
 819                 except ExtractorError as e:
 820                     # YouTube servers may return errors we want to retry on in a 200 OK response
 821                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 822                     if 'unknown error' in e.msg.lower():
 823                         last_error = e.msg
 824                         continue
 825                     if fatal:
 826                         raise
 827                     self.report_warning(error_to_compat_str(e))
 828                     return
 829                 if not check_get_keys or dict_get(response, check_get_keys):
 830                     break
 831                 # Youtube sometimes sends incomplete data
 832                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 833                 last_error = 'Incomplete data received'
 834                 if count >= retries:
 835                     if fatal:
 836                         raise ExtractorError(last_error)
 837                     else:
 838                         self.report_warning(last_error)
 839                         return
 840         return response
 841
 842     @staticmethod
 843     def is_music_url(url):
 844         return re.match(r'https?://music\.youtube\.com/', url) is not None
 845
 846     def _extract_video(self, renderer):
 847         video_id = renderer.get('videoId')
 848         title = self._get_text(renderer, 'title')
 849         description = self._get_text(renderer, 'descriptionSnippet')
 850         duration = parse_duration(self._get_text(
 851             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 852         if duration is None:
 853             duration = parse_duration(self._search_regex(
 854                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 855                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 856                 video_id, default=None, group='duration'))
 857
 858         view_count = self._get_count(renderer, 'viewCountText')
 859
 860         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 861         channel_id = traverse_obj(
 862             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 863             expected_type=str, get_all=False)
 864         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 865         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 866         overlay_style = traverse_obj(
 867             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 868             get_all=False, expected_type=str)
 869         badges = self._extract_badges(renderer)
 870         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 871         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 872             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 873             expected_type=str)) or ''
 874         url = f'https://www.youtube.com/watch?v={video_id}'
 875         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 876             url = f'https://www.youtube.com/shorts/{video_id}'
 877
 878         return {
 879             '_type': 'url',
 880             'ie_key': YoutubeIE.ie_key(),
 881             'id': video_id,
 882             'url': url,
 883             'title': title,
 884             'description': description,
 885             'duration': duration,
 886             'view_count': view_count,
 887             'uploader': uploader,
 888             'channel_id': channel_id,
 889             'thumbnails': thumbnails,
 890             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 891                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 892                             else None),
 893             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 894                             else 'was_live' if 'streamed' in time_text.lower()
 895                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 896                             else None),
 897             'release_timestamp': scheduled_timestamp,
 898             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 899         }
 900
 901
 902 class YoutubeIE(YoutubeBaseInfoExtractor):
 903     IE_DESC = 'YouTube'
 904     _VALID_URL = r"""(?x)^
 905                      (
 906                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 907                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 908                             (?:www\.)?deturl\.com/www\.youtube\.com|
 909                             (?:www\.)?pwnyoutube\.com|
 910                             (?:www\.)?hooktube\.com|
 911                             (?:www\.)?yourepeat\.com|
 912                             tube\.majestyc\.net|
 913                             %(invidious)s|
 914                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 915                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 916                          (?:                                                  # the various things that can precede the ID:
 917                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 918                              |(?:                                             # or the v= param in all its forms
 919                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 920                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 921                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 922                                  v=
 923                              )
 924                          ))
 925                          |(?:
 926                             youtu\.be|                                        # just youtu.be/xxxx
 927                             vid\.plus|                                        # or vid.plus/xxxx
 928                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 929                             %(invidious)s
 930                          )/
 931                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 932                          )
 933                      )?                                                       # all until now is optional -> you can pass the naked ID
 934                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 935                      (?(1).+)?                                                # if we found the ID, everything can follow
 936                      (?:\#|$)""" % {
 937         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 938     }
 939     _PLAYER_INFO_RE = (
 940         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 941         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 942         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 943     )
 944     _formats = {
 945         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 946         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 947         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 948         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 949         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 950         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 951         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 953         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 954         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 955         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 956         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 957         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 958         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 959         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 960         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 961         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 962         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 963
 964
 965         # 3D videos
 966         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 967         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 968         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 969         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 970         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 971         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 972         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 973
 974         # Apple HTTP Live Streaming
 975         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 976         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 977         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 978         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 979         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 980         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 981         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 982         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 983
 984         # DASH mp4 video
 985         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 986         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 988         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 989         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 990         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 991         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 992         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 993         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 994         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 995         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 996         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 997
 998         # Dash mp4 audio
 999         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1000         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1001         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1002         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1003         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1004         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1005         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1006
1007         # Dash webm
1008         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1009         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1010         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1011         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1012         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1013         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1014         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1015         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1016         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1017         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1018         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1020         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1021         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1022         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1023         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1024         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1025         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1026         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1027         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1028         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1029         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1030
1031         # Dash webm audio
1032         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1033         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1034
1035         # Dash webm audio with opus inside
1036         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1037         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1038         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1039
1040         # RTMP (unnamed)
1041         '_rtmp': {'protocol': 'rtmp'},
1042
1043         # av01 video only formats sometimes served with "unknown" codecs
1044         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1045         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1046         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1047         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1048         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1049         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1050         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1051         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1052     }
1053     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1054
1055     _GEO_BYPASS = False
1056
1057     IE_NAME = 'youtube'
1058     _TESTS = [
1059         {
1060             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1061             'info_dict': {
1062                 'id': 'BaW_jenozKc',
1063                 'ext': 'mp4',
1064                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1065                 'uploader': 'Philipp Hagemeister',
1066                 'uploader_id': 'phihag',
1067                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1068                 'channel': 'Philipp Hagemeister',
1069                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1070                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1071                 'upload_date': '20121002',
1072                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1073                 'categories': ['Science & Technology'],
1074                 'tags': ['youtube-dl'],
1075                 'duration': 10,
1076                 'view_count': int,
1077                 'like_count': int,
1078                 'availability': 'public',
1079                 'playable_in_embed': True,
1080                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1081                 'live_status': 'not_live',
1082                 'age_limit': 0,
1083                 'start_time': 1,
1084                 'end_time': 9,
1085                 'channel_follower_count': int
1086             }
1087         },
1088         {
1089             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1090             'note': 'Embed-only video (#1746)',
1091             'info_dict': {
1092                 'id': 'yZIXLfi8CZQ',
1093                 'ext': 'mp4',
1094                 'upload_date': '20120608',
1095                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1096                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1097                 'uploader': 'SET India',
1098                 'uploader_id': 'setindia',
1099                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1100                 'age_limit': 18,
1101             },
1102             'skip': 'Private video',
1103         },
1104         {
1105             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1106             'note': 'Use the first video ID in the URL',
1107             'info_dict': {
1108                 'id': 'BaW_jenozKc',
1109                 'ext': 'mp4',
1110                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1111                 'uploader': 'Philipp Hagemeister',
1112                 'uploader_id': 'phihag',
1113                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1114                 'channel': 'Philipp Hagemeister',
1115                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1116                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1117                 'upload_date': '20121002',
1118                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1119                 'categories': ['Science & Technology'],
1120                 'tags': ['youtube-dl'],
1121                 'duration': 10,
1122                 'view_count': int,
1123                 'like_count': int,
1124                 'availability': 'public',
1125                 'playable_in_embed': True,
1126                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1127                 'live_status': 'not_live',
1128                 'age_limit': 0,
1129                 'channel_follower_count': int
1130             },
1131             'params': {
1132                 'skip_download': True,
1133             },
1134         },
1135         {
1136             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1137             'note': '256k DASH audio (format 141) via DASH manifest',
1138             'info_dict': {
1139                 'id': 'a9LDPn-MO4I',
1140                 'ext': 'm4a',
1141                 'upload_date': '20121002',
1142                 'uploader_id': '8KVIDEO',
1143                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1144                 'description': '',
1145                 'uploader': '8KVIDEO',
1146                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1147             },
1148             'params': {
1149                 'youtube_include_dash_manifest': True,
1150                 'format': '141',
1151             },
1152             'skip': 'format 141 not served anymore',
1153         },
1154         # DASH manifest with encrypted signature
1155         {
1156             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1157             'info_dict': {
1158                 'id': 'IB3lcPjvWLA',
1159                 'ext': 'm4a',
1160                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1161                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1162                 'duration': 244,
1163                 'uploader': 'AfrojackVEVO',
1164                 'uploader_id': 'AfrojackVEVO',
1165                 'upload_date': '20131011',
1166                 'abr': 129.495,
1167                 'like_count': int,
1168                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1169                 'playable_in_embed': True,
1170                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1171                 'view_count': int,
1172                 'track': 'The Spark',
1173                 'live_status': 'not_live',
1174                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1175                 'channel': 'Afrojack',
1176                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1177                 'tags': 'count:19',
1178                 'availability': 'public',
1179                 'categories': ['Music'],
1180                 'age_limit': 0,
1181                 'alt_title': 'The Spark',
1182                 'channel_follower_count': int
1183             },
1184             'params': {
1185                 'youtube_include_dash_manifest': True,
1186                 'format': '141/bestaudio[ext=m4a]',
1187             },
1188         },
1189         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1190         {
1191             'note': 'Embed allowed age-gate video',
1192             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1193             'info_dict': {
1194                 'id': 'HtVdAasjOgU',
1195                 'ext': 'mp4',
1196                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1197                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1198                 'duration': 142,
1199                 'uploader': 'The Witcher',
1200                 'uploader_id': 'WitcherGame',
1201                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1202                 'upload_date': '20140605',
1203                 'age_limit': 18,
1204                 'categories': ['Gaming'],
1205                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1206                 'availability': 'needs_auth',
1207                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1208                 'like_count': int,
1209                 'channel': 'The Witcher',
1210                 'live_status': 'not_live',
1211                 'tags': 'count:17',
1212                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1213                 'playable_in_embed': True,
1214                 'view_count': int,
1215                 'channel_follower_count': int
1216             },
1217         },
1218         {
1219             'note': 'Age-gate video with embed allowed in public site',
1220             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1221             'info_dict': {
1222                 'id': 'HsUATh_Nc2U',
1223                 'ext': 'mp4',
1224                 'title': 'Godzilla 2 (Official Video)',
1225                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1226                 'upload_date': '20200408',
1227                 'uploader_id': 'FlyingKitty900',
1228                 'uploader': 'FlyingKitty',
1229                 'age_limit': 18,
1230                 'availability': 'needs_auth',
1231                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1232                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1233                 'channel': 'FlyingKitty',
1234                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1235                 'view_count': int,
1236                 'categories': ['Entertainment'],
1237                 'live_status': 'not_live',
1238                 'tags': ['Flyingkitty', 'godzilla 2'],
1239                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1240                 'like_count': int,
1241                 'duration': 177,
1242                 'playable_in_embed': True,
1243                 'channel_follower_count': int
1244             },
1245         },
1246         {
1247             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1248             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1249             'info_dict': {
1250                 'id': 'Tq92D6wQ1mg',
1251                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1252                 'ext': 'mp4',
1253                 'upload_date': '20191228',
1254                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1255                 'uploader': 'Projekt Melody',
1256                 'description': 'md5:17eccca93a786d51bc67646756894066',
1257                 'age_limit': 18,
1258                 'like_count': int,
1259                 'availability': 'needs_auth',
1260                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1261                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1262                 'view_count': int,
1263                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1264                 'channel': 'Projekt Melody',
1265                 'live_status': 'not_live',
1266                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1267                 'playable_in_embed': True,
1268                 'categories': ['Entertainment'],
1269                 'duration': 106,
1270                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1271                 'channel_follower_count': int
1272             },
1273         },
1274         {
1275             'note': 'Non-Agegated non-embeddable video',
1276             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1277             'info_dict': {
1278                 'id': 'MeJVWBSsPAY',
1279                 'ext': 'mp4',
1280                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1281                 'uploader': 'Herr Lurik',
1282                 'uploader_id': 'st3in234',
1283                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1284                 'upload_date': '20130730',
1285                 'track': 'Such mich find mich',
1286                 'age_limit': 0,
1287                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1288                 'like_count': int,
1289                 'playable_in_embed': False,
1290                 'creator': 'OOMPH!',
1291                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1292                 'view_count': int,
1293                 'alt_title': 'Such mich find mich',
1294                 'duration': 210,
1295                 'channel': 'Herr Lurik',
1296                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1297                 'categories': ['Music'],
1298                 'availability': 'public',
1299                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1300                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1301                 'live_status': 'not_live',
1302                 'artist': 'OOMPH!',
1303                 'channel_follower_count': int
1304             },
1305         },
1306         {
1307             'note': 'Non-bypassable age-gated video',
1308             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1309             'only_matching': True,
1310         },
1311         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1312         # YouTube Red ad is not captured for creator
1313         {
1314             'url': '__2ABJjxzNo',
1315             'info_dict': {
1316                 'id': '__2ABJjxzNo',
1317                 'ext': 'mp4',
1318                 'duration': 266,
1319                 'upload_date': '20100430',
1320                 'uploader_id': 'deadmau5',
1321                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1322                 'creator': 'deadmau5',
1323                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1324                 'uploader': 'deadmau5',
1325                 'title': 'Deadmau5 - Some Chords (HD)',
1326                 'alt_title': 'Some Chords',
1327                 'availability': 'public',
1328                 'tags': 'count:14',
1329                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1330                 'view_count': int,
1331                 'live_status': 'not_live',
1332                 'channel': 'deadmau5',
1333                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1334                 'like_count': int,
1335                 'track': 'Some Chords',
1336                 'artist': 'deadmau5',
1337                 'playable_in_embed': True,
1338                 'age_limit': 0,
1339                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1340                 'categories': ['Music'],
1341                 'album': 'Some Chords',
1342                 'channel_follower_count': int
1343             },
1344             'expected_warnings': [
1345                 'DASH manifest missing',
1346             ]
1347         },
1348         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1349         {
1350             'url': 'lqQg6PlCWgI',
1351             'info_dict': {
1352                 'id': 'lqQg6PlCWgI',
1353                 'ext': 'mp4',
1354                 'duration': 6085,
1355                 'upload_date': '20150827',
1356                 'uploader_id': 'olympic',
1357                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1358                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1359                 'uploader': 'Olympics',
1360                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1361                 'like_count': int,
1362                 'release_timestamp': 1343767800,
1363                 'playable_in_embed': True,
1364                 'categories': ['Sports'],
1365                 'release_date': '20120731',
1366                 'channel': 'Olympics',
1367                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1368                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1369                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1370                 'age_limit': 0,
1371                 'availability': 'public',
1372                 'live_status': 'was_live',
1373                 'view_count': int,
1374                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1375                 'channel_follower_count': int
1376             },
1377             'params': {
1378                 'skip_download': 'requires avconv',
1379             }
1380         },
1381         # Non-square pixels
1382         {
1383             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1384             'info_dict': {
1385                 'id': '_b-2C3KPAM0',
1386                 'ext': 'mp4',
1387                 'stretched_ratio': 16 / 9.,
1388                 'duration': 85,
1389                 'upload_date': '20110310',
1390                 'uploader_id': 'AllenMeow',
1391                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1392                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1393                 'uploader': '孫ᄋᄅ',
1394                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1395                 'playable_in_embed': True,
1396                 'channel': '孫ᄋᄅ',
1397                 'age_limit': 0,
1398                 'tags': 'count:11',
1399                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1400                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1401                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1402                 'view_count': int,
1403                 'categories': ['People & Blogs'],
1404                 'like_count': int,
1405                 'live_status': 'not_live',
1406                 'availability': 'unlisted',
1407                 'channel_follower_count': int
1408             },
1409         },
1410         # url_encoded_fmt_stream_map is empty string
1411         {
1412             'url': 'qEJwOuvDf7I',
1413             'info_dict': {
1414                 'id': 'qEJwOuvDf7I',
1415                 'ext': 'webm',
1416                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1417                 'description': '',
1418                 'upload_date': '20150404',
1419                 'uploader_id': 'spbelect',
1420                 'uploader': 'Наблюдатели Петербурга',
1421             },
1422             'params': {
1423                 'skip_download': 'requires avconv',
1424             },
1425             'skip': 'This live event has ended.',
1426         },
1427         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1428         {
1429             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1430             'info_dict': {
1431                 'id': 'FIl7x6_3R5Y',
1432                 'ext': 'webm',
1433                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1434                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1435                 'duration': 220,
1436                 'upload_date': '20150625',
1437                 'uploader_id': 'dorappi2000',
1438                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1439                 'uploader': 'dorappi2000',
1440                 'formats': 'mincount:31',
1441             },
1442             'skip': 'not actual anymore',
1443         },
1444         # DASH manifest with segment_list
1445         {
1446             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1447             'md5': '8ce563a1d667b599d21064e982ab9e31',
1448             'info_dict': {
1449                 'id': 'CsmdDsKjzN8',
1450                 'ext': 'mp4',
1451                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1452                 'uploader': 'Airtek',
1453                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1454                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1455                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1456             },
1457             'params': {
1458                 'youtube_include_dash_manifest': True,
1459                 'format': '135',  # bestvideo
1460             },
1461             'skip': 'This live event has ended.',
1462         },
1463         {
1464             # Multifeed videos (multiple cameras), URL is for Main Camera
1465             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1466             'info_dict': {
1467                 'id': 'jvGDaLqkpTg',
1468                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1469                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1470             },
1471             'playlist': [{
1472                 'info_dict': {
1473                     'id': 'jvGDaLqkpTg',
1474                     'ext': 'mp4',
1475                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1476                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1477                     'duration': 10643,
1478                     'upload_date': '20161111',
1479                     'uploader': 'Team PGP',
1480                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1481                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1482                 },
1483             }, {
1484                 'info_dict': {
1485                     'id': '3AKt1R1aDnw',
1486                     'ext': 'mp4',
1487                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1488                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1489                     'duration': 10991,
1490                     'upload_date': '20161111',
1491                     'uploader': 'Team PGP',
1492                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1493                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1494                 },
1495             }, {
1496                 'info_dict': {
1497                     'id': 'RtAMM00gpVc',
1498                     'ext': 'mp4',
1499                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1500                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1501                     'duration': 10995,
1502                     'upload_date': '20161111',
1503                     'uploader': 'Team PGP',
1504                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1505                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1506                 },
1507             }, {
1508                 'info_dict': {
1509                     'id': '6N2fdlP3C5U',
1510                     'ext': 'mp4',
1511                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1512                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1513                     'duration': 10990,
1514                     'upload_date': '20161111',
1515                     'uploader': 'Team PGP',
1516                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1517                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1518                 },
1519             }],
1520             'params': {
1521                 'skip_download': True,
1522             },
1523             'skip': 'Not multifeed anymore',
1524         },
1525         {
1526             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1527             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1528             'info_dict': {
1529                 'id': 'gVfLd0zydlo',
1530                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1531             },
1532             'playlist_count': 2,
1533             'skip': 'Not multifeed anymore',
1534         },
1535         {
1536             'url': 'https://vid.plus/FlRa-iH7PGw',
1537             'only_matching': True,
1538         },
1539         {
1540             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1541             'only_matching': True,
1542         },
1543         {
1544             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1545             # Also tests cut-off URL expansion in video description (see
1546             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1547             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1548             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1549             'info_dict': {
1550                 'id': 'lsguqyKfVQg',
1551                 'ext': 'mp4',
1552                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1553                 'alt_title': 'Dark Walk',
1554                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1555                 'duration': 133,
1556                 'upload_date': '20151119',
1557                 'uploader_id': 'IronSoulElf',
1558                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1559                 'uploader': 'IronSoulElf',
1560                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1561                 'track': 'Dark Walk',
1562                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1563                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1564                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1565                 'categories': ['Film & Animation'],
1566                 'view_count': int,
1567                 'live_status': 'not_live',
1568                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1569                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1570                 'tags': 'count:13',
1571                 'availability': 'public',
1572                 'channel': 'IronSoulElf',
1573                 'playable_in_embed': True,
1574                 'like_count': int,
1575                 'age_limit': 0,
1576                 'channel_follower_count': int
1577             },
1578             'params': {
1579                 'skip_download': True,
1580             },
1581         },
1582         {
1583             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1584             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1585             'only_matching': True,
1586         },
1587         {
1588             # Video with yt:stretch=17:0
1589             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1590             'info_dict': {
1591                 'id': 'Q39EVAstoRM',
1592                 'ext': 'mp4',
1593                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1594                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1595                 'upload_date': '20151107',
1596                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1597                 'uploader': 'CH GAMER DROID',
1598             },
1599             'params': {
1600                 'skip_download': True,
1601             },
1602             'skip': 'This video does not exist.',
1603         },
1604         {
1605             # Video with incomplete 'yt:stretch=16:'
1606             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1607             'only_matching': True,
1608         },
1609         {
1610             # Video licensed under Creative Commons
1611             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1612             'info_dict': {
1613                 'id': 'M4gD1WSo5mA',
1614                 'ext': 'mp4',
1615                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1616                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1617                 'duration': 721,
1618                 'upload_date': '20150128',
1619                 'uploader_id': 'BerkmanCenter',
1620                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1621                 'uploader': 'The Berkman Klein Center for Internet & Society',
1622                 'license': 'Creative Commons Attribution license (reuse allowed)',
1623                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1624                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1625                 'like_count': int,
1626                 'age_limit': 0,
1627                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1628                 'channel': 'The Berkman Klein Center for Internet & Society',
1629                 'availability': 'public',
1630                 'view_count': int,
1631                 'categories': ['Education'],
1632                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1633                 'live_status': 'not_live',
1634                 'playable_in_embed': True,
1635                 'channel_follower_count': int
1636             },
1637             'params': {
1638                 'skip_download': True,
1639             },
1640         },
1641         {
1642             # Channel-like uploader_url
1643             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1644             'info_dict': {
1645                 'id': 'eQcmzGIKrzg',
1646                 'ext': 'mp4',
1647                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1648                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1649                 'duration': 4060,
1650                 'upload_date': '20151120',
1651                 'uploader': 'Bernie Sanders',
1652                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1653                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1654                 'license': 'Creative Commons Attribution license (reuse allowed)',
1655                 'playable_in_embed': True,
1656                 'tags': 'count:12',
1657                 'like_count': int,
1658                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1659                 'age_limit': 0,
1660                 'availability': 'public',
1661                 'categories': ['News & Politics'],
1662                 'channel': 'Bernie Sanders',
1663                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1664                 'view_count': int,
1665                 'live_status': 'not_live',
1666                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1667                 'channel_follower_count': int
1668             },
1669             'params': {
1670                 'skip_download': True,
1671             },
1672         },
1673         {
1674             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1675             'only_matching': True,
1676         },
1677         {
1678             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1679             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1680             'only_matching': True,
1681         },
1682         {
1683             # Rental video preview
1684             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1685             'info_dict': {
1686                 'id': 'uGpuVWrhIzE',
1687                 'ext': 'mp4',
1688                 'title': 'Piku - Trailer',
1689                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1690                 'upload_date': '20150811',
1691                 'uploader': 'FlixMatrix',
1692                 'uploader_id': 'FlixMatrixKaravan',
1693                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1694                 'license': 'Standard YouTube License',
1695             },
1696             'params': {
1697                 'skip_download': True,
1698             },
1699             'skip': 'This video is not available.',
1700         },
1701         {
1702             # YouTube Red video with episode data
1703             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1704             'info_dict': {
1705                 'id': 'iqKdEhx-dD4',
1706                 'ext': 'mp4',
1707                 'title': 'Isolation - Mind Field (Ep 1)',
1708                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1709                 'duration': 2085,
1710                 'upload_date': '20170118',
1711                 'uploader': 'Vsauce',
1712                 'uploader_id': 'Vsauce',
1713                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1714                 'series': 'Mind Field',
1715                 'season_number': 1,
1716                 'episode_number': 1,
1717                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1718                 'tags': 'count:12',
1719                 'view_count': int,
1720                 'availability': 'public',
1721                 'age_limit': 0,
1722                 'channel': 'Vsauce',
1723                 'episode': 'Episode 1',
1724                 'categories': ['Entertainment'],
1725                 'season': 'Season 1',
1726                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1727                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1728                 'like_count': int,
1729                 'playable_in_embed': True,
1730                 'live_status': 'not_live',
1731                 'channel_follower_count': int
1732             },
1733             'params': {
1734                 'skip_download': True,
1735             },
1736             'expected_warnings': [
1737                 'Skipping DASH manifest',
1738             ],
1739         },
1740         {
1741             # The following content has been identified by the YouTube community
1742             # as inappropriate or offensive to some audiences.
1743             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1744             'info_dict': {
1745                 'id': '6SJNVb0GnPI',
1746                 'ext': 'mp4',
1747                 'title': 'Race Differences in Intelligence',
1748                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1749                 'duration': 965,
1750                 'upload_date': '20140124',
1751                 'uploader': 'New Century Foundation',
1752                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1753                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1754             },
1755             'params': {
1756                 'skip_download': True,
1757             },
1758             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1759         },
1760         {
1761             # itag 212
1762             'url': '1t24XAntNCY',
1763             'only_matching': True,
1764         },
1765         {
1766             # geo restricted to JP
1767             'url': 'sJL6WA-aGkQ',
1768             'only_matching': True,
1769         },
1770         {
1771             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1772             'only_matching': True,
1773         },
1774         {
1775             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1776             'only_matching': True,
1777         },
1778         {
1779             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1780             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1781             'only_matching': True,
1782         },
1783         {
1784             # DRM protected
1785             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1786             'only_matching': True,
1787         },
1788         {
1789             # Video with unsupported adaptive stream type formats
1790             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1791             'info_dict': {
1792                 'id': 'Z4Vy8R84T1U',
1793                 'ext': 'mp4',
1794                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1795                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1796                 'duration': 433,
1797                 'upload_date': '20130923',
1798                 'uploader': 'Amelia Putri Harwita',
1799                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1800                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1801                 'formats': 'maxcount:10',
1802             },
1803             'params': {
1804                 'skip_download': True,
1805                 'youtube_include_dash_manifest': False,
1806             },
1807             'skip': 'not actual anymore',
1808         },
1809         {
1810             # Youtube Music Auto-generated description
1811             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1812             'info_dict': {
1813                 'id': 'MgNrAu2pzNs',
1814                 'ext': 'mp4',
1815                 'title': 'Voyeur Girl',
1816                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1817                 'upload_date': '20190312',
1818                 'uploader': 'Stephen - Topic',
1819                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1820                 'artist': 'Stephen',
1821                 'track': 'Voyeur Girl',
1822                 'album': 'it\'s too much love to know my dear',
1823                 'release_date': '20190313',
1824                 'release_year': 2019,
1825                 'alt_title': 'Voyeur Girl',
1826                 'view_count': int,
1827                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1828                 'playable_in_embed': True,
1829                 'like_count': int,
1830                 'categories': ['Music'],
1831                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1832                 'channel': 'Stephen',
1833                 'availability': 'public',
1834                 'creator': 'Stephen',
1835                 'duration': 169,
1836                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1837                 'age_limit': 0,
1838                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1839                 'tags': 'count:11',
1840                 'live_status': 'not_live',
1841                 'channel_follower_count': int
1842             },
1843             'params': {
1844                 'skip_download': True,
1845             },
1846         },
1847         {
1848             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1849             'only_matching': True,
1850         },
1851         {
1852             # invalid -> valid video id redirection
1853             'url': 'DJztXj2GPfl',
1854             'info_dict': {
1855                 'id': 'DJztXj2GPfk',
1856                 'ext': 'mp4',
1857                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1858                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1859                 'upload_date': '20090125',
1860                 'uploader': 'Prochorowka',
1861                 'uploader_id': 'Prochorowka',
1862                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1863                 'artist': 'Panjabi MC',
1864                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1865                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1866             },
1867             'params': {
1868                 'skip_download': True,
1869             },
1870             'skip': 'Video unavailable',
1871         },
1872         {
1873             # empty description results in an empty string
1874             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1875             'info_dict': {
1876                 'id': 'x41yOUIvK2k',
1877                 'ext': 'mp4',
1878                 'title': 'IMG 3456',
1879                 'description': '',
1880                 'upload_date': '20170613',
1881                 'uploader_id': 'ElevageOrVert',
1882                 'uploader': 'ElevageOrVert',
1883                 'view_count': int,
1884                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1885                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1886                 'like_count': int,
1887                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1888                 'tags': [],
1889                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1890                 'availability': 'public',
1891                 'age_limit': 0,
1892                 'categories': ['Pets & Animals'],
1893                 'duration': 7,
1894                 'playable_in_embed': True,
1895                 'live_status': 'not_live',
1896                 'channel': 'ElevageOrVert',
1897                 'channel_follower_count': int
1898             },
1899             'params': {
1900                 'skip_download': True,
1901             },
1902         },
1903         {
1904             # with '};' inside yt initial data (see [1])
1905             # see [2] for an example with '};' inside ytInitialPlayerResponse
1906             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1907             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1908             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1909             'info_dict': {
1910                 'id': 'CHqg6qOn4no',
1911                 'ext': 'mp4',
1912                 'title': 'Part 77   Sort a list of simple types in c#',
1913                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1914                 'upload_date': '20130831',
1915                 'uploader_id': 'kudvenkat',
1916                 'uploader': 'kudvenkat',
1917                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1918                 'like_count': int,
1919                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1920                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1921                 'live_status': 'not_live',
1922                 'categories': ['Education'],
1923                 'availability': 'public',
1924                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1925                 'tags': 'count:12',
1926                 'playable_in_embed': True,
1927                 'age_limit': 0,
1928                 'view_count': int,
1929                 'duration': 522,
1930                 'channel': 'kudvenkat',
1931                 'channel_follower_count': int
1932             },
1933             'params': {
1934                 'skip_download': True,
1935             },
1936         },
1937         {
1938             # another example of '};' in ytInitialData
1939             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1940             'only_matching': True,
1941         },
1942         {
1943             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1944             'only_matching': True,
1945         },
1946         {
1947             # https://github.com/ytdl-org/youtube-dl/pull/28094
1948             'url': 'OtqTfy26tG0',
1949             'info_dict': {
1950                 'id': 'OtqTfy26tG0',
1951                 'ext': 'mp4',
1952                 'title': 'Burn Out',
1953                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1954                 'upload_date': '20141120',
1955                 'uploader': 'The Cinematic Orchestra - Topic',
1956                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1957                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1958                 'artist': 'The Cinematic Orchestra',
1959                 'track': 'Burn Out',
1960                 'album': 'Every Day',
1961                 'like_count': int,
1962                 'live_status': 'not_live',
1963                 'alt_title': 'Burn Out',
1964                 'duration': 614,
1965                 'age_limit': 0,
1966                 'view_count': int,
1967                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1968                 'creator': 'The Cinematic Orchestra',
1969                 'channel': 'The Cinematic Orchestra',
1970                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1971                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1972                 'availability': 'public',
1973                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1974                 'categories': ['Music'],
1975                 'playable_in_embed': True,
1976                 'channel_follower_count': int
1977             },
1978             'params': {
1979                 'skip_download': True,
1980             },
1981         },
1982         {
1983             # controversial video, only works with bpctr when authenticated with cookies
1984             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1985             'only_matching': True,
1986         },
1987         {
1988             # controversial video, requires bpctr/contentCheckOk
1989             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1990             'info_dict': {
1991                 'id': 'SZJvDhaSDnc',
1992                 'ext': 'mp4',
1993                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1994                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1995                 'uploader': 'CBS Mornings',
1996                 'uploader_id': 'CBSThisMorning',
1997                 'upload_date': '20140716',
1998                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1999                 'duration': 170,
2000                 'categories': ['News & Politics'],
2001                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2002                 'view_count': int,
2003                 'channel': 'CBS Mornings',
2004                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2005                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2006                 'age_limit': 18,
2007                 'availability': 'needs_auth',
2008                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2009                 'like_count': int,
2010                 'live_status': 'not_live',
2011                 'playable_in_embed': True,
2012                 'channel_follower_count': int
2013             }
2014         },
2015         {
2016             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2017             'url': 'cBvYw8_A0vQ',
2018             'info_dict': {
2019                 'id': 'cBvYw8_A0vQ',
2020                 'ext': 'mp4',
2021                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2022                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2023                 'upload_date': '20201120',
2024                 'uploader': 'Walk around Japan',
2025                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2026                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2027                 'duration': 1456,
2028                 'categories': ['Travel & Events'],
2029                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2030                 'view_count': int,
2031                 'channel': 'Walk around Japan',
2032                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2033                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2034                 'age_limit': 0,
2035                 'availability': 'public',
2036                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2037                 'live_status': 'not_live',
2038                 'playable_in_embed': True,
2039                 'channel_follower_count': int
2040             },
2041             'params': {
2042                 'skip_download': True,
2043             },
2044         }, {
2045             # Has multiple audio streams
2046             'url': 'WaOKSUlf4TM',
2047             'only_matching': True
2048         }, {
2049             # Requires Premium: has format 141 when requested using YTM url
2050             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2051             'only_matching': True
2052         }, {
2053             # multiple subtitles with same lang_code
2054             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2055             'only_matching': True,
2056         }, {
2057             # Force use android client fallback
2058             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2059             'info_dict': {
2060                 'id': 'YOelRv7fMxY',
2061                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2062                 'ext': '3gp',
2063                 'upload_date': '20210624',
2064                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2065                 'uploader': 'colinfurze',
2066                 'uploader_id': 'colinfurze',
2067                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2068                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2069                 'duration': 596,
2070                 'categories': ['Entertainment'],
2071                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2072                 'view_count': int,
2073                 'channel': 'colinfurze',
2074                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2075                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2076                 'age_limit': 0,
2077                 'availability': 'public',
2078                 'like_count': int,
2079                 'live_status': 'not_live',
2080                 'playable_in_embed': True,
2081                 'channel_follower_count': int
2082             },
2083             'params': {
2084                 'format': '17',  # 3gp format available on android
2085                 'extractor_args': {'youtube': {'player_client': ['android']}},
2086             },
2087         },
2088         {
2089             # Skip download of additional client configs (remix client config in this case)
2090             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2091             'only_matching': True,
2092             'params': {
2093                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2094             },
2095         }, {
2096             # shorts
2097             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2098             'only_matching': True,
2099         }, {
2100             'note': 'Storyboards',
2101             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2102             'info_dict': {
2103                 'id': '5KLPxDtMqe8',
2104                 'ext': 'mhtml',
2105                 'format_id': 'sb0',
2106                 'title': 'Your Brain is Plastic',
2107                 'uploader_id': 'scishow',
2108                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2109                 'upload_date': '20140324',
2110                 'uploader': 'SciShow',
2111                 'like_count': int,
2112                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2113                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2114                 'view_count': int,
2115                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2116                 'playable_in_embed': True,
2117                 'tags': 'count:12',
2118                 'uploader_url': 'http://www.youtube.com/user/scishow',
2119                 'availability': 'public',
2120                 'channel': 'SciShow',
2121                 'live_status': 'not_live',
2122                 'duration': 248,
2123                 'categories': ['Education'],
2124                 'age_limit': 0,
2125                 'channel_follower_count': int
2126             }, 'params': {'format': 'mhtml', 'skip_download': True}
2127         }, {
2128             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2129             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2130             'info_dict': {
2131                 'id': '2NUZ8W2llS4',
2132                 'ext': 'mp4',
2133                 'title': 'The NP that test your phone performance 🙂',
2134                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2135                 'uploader': 'Leon Nguyen',
2136                 'uploader_id': 'VNSXIII',
2137                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2138                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2139                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2140                 'duration': 21,
2141                 'view_count': int,
2142                 'age_limit': 0,
2143                 'categories': ['Gaming'],
2144                 'tags': 'count:23',
2145                 'playable_in_embed': True,
2146                 'live_status': 'not_live',
2147                 'upload_date': '20220103',
2148                 'like_count': int,
2149                 'availability': 'public',
2150                 'channel': 'Leon Nguyen',
2151                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2152                 'channel_follower_count': int
2153             }
2154         }, {
2155             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2156             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2157             'info_dict': {
2158                 'id': 'mzZzzBU6lrM',
2159                 'ext': 'mp4',
2160                 'title': 'I Met GeorgeNotFound In Real Life...',
2161                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2162                 'uploader': 'Quackity',
2163                 'uploader_id': 'QuackityHQ',
2164                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2165                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2166                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2167                 'duration': 955,
2168                 'view_count': int,
2169                 'age_limit': 0,
2170                 'categories': ['Entertainment'],
2171                 'tags': 'count:26',
2172                 'playable_in_embed': True,
2173                 'live_status': 'not_live',
2174                 'release_timestamp': 1641172509,
2175                 'release_date': '20220103',
2176                 'upload_date': '20220103',
2177                 'like_count': int,
2178                 'availability': 'public',
2179                 'channel': 'Quackity',
2180                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2181                 'channel_follower_count': int
2182             }
2183         },
2184         {   # continuous livestream. Microformat upload date should be preferred.
2185             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2186             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2187             'info_dict': {
2188                 'id': 'kgx4WGK0oNU',
2189                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2190                 'ext': 'mp4',
2191                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2192                 'availability': 'public',
2193                 'age_limit': 0,
2194                 'release_timestamp': 1637975704,
2195                 'upload_date': '20210619',
2196                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2197                 'live_status': 'is_live',
2198                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2199                 'uploader': '阿鲍Abao',
2200                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2201                 'channel': 'Abao in Tokyo',
2202                 'channel_follower_count': int,
2203                 'release_date': '20211127',
2204                 'tags': 'count:39',
2205                 'categories': ['People & Blogs'],
2206                 'like_count': int,
2207                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2208                 'view_count': int,
2209                 'playable_in_embed': True,
2210                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2211             },
2212             'params': {'skip_download': True}
2213         }, {
2214             # Story. Requires specific player params to work.
2215             # Note: stories get removed after some period of time
2216             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2217             'info_dict': {
2218                 'id': 'vv8qTUWmulI',
2219                 'ext': 'mp4',
2220                 'availability': 'unlisted',
2221                 'view_count': int,
2222                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2223                 'upload_date': '20220526',
2224                 'categories': ['Education'],
2225                 'title': 'Story',
2226                 'channel': 'IT\'S HISTORY',
2227                 'description': '',
2228                 'uploader_id': 'BlastfromthePast',
2229                 'duration': 12,
2230                 'uploader': 'IT\'S HISTORY',
2231                 'playable_in_embed': True,
2232                 'age_limit': 0,
2233                 'live_status': 'not_live',
2234                 'tags': [],
2235                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2236                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2237                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2238             }
2239         }, {
2240             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2241             'info_dict': {
2242                 'id': 'tjjjtzRLHvA',
2243                 'ext': 'mp4',
2244                 'title': 'ハッシュタグ無し };if window.ytcsi',
2245                 'upload_date': '20220323',
2246                 'like_count': int,
2247                 'availability': 'unlisted',
2248                 'channel': 'nao20010128nao',
2249                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2250                 'age_limit': 0,
2251                 'uploader': 'nao20010128nao',
2252                 'uploader_id': 'nao20010128nao',
2253                 'categories': ['Music'],
2254                 'view_count': int,
2255                 'description': '',
2256                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2257                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2258                 'live_status': 'not_live',
2259                 'playable_in_embed': True,
2260                 'channel_follower_count': int,
2261                 'duration': 6,
2262                 'tags': [],
2263                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2264             }
2265         }
2266     ]
2267
2268     @classmethod
2269     def suitable(cls, url):
2270         from ..utils import parse_qs
2271
2272         qs = parse_qs(url)
2273         if qs.get('list', [None])[0]:
2274             return False
2275         return super().suitable(url)
2276
2277     def __init__(self, *args, **kwargs):
2278         super().__init__(*args, **kwargs)
2279         self._code_cache = {}
2280         self._player_cache = {}
2281
2282     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2283         lock = threading.Lock()
2284
2285         is_live = True
2286         start_time = time.time()
2287         formats = [f for f in formats if f.get('is_from_start')]
2288
2289         def refetch_manifest(format_id, delay):
2290             nonlocal formats, start_time, is_live
2291             if time.time() <= start_time + delay:
2292                 return
2293
2294             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2295             video_details = traverse_obj(
2296                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2297             microformats = traverse_obj(
2298                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2299                 expected_type=dict, default=[])
2300             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2301             start_time = time.time()
2302
2303         def mpd_feed(format_id, delay):
2304             """
2305             @returns (manifest_url, manifest_stream_number, is_live) or None
2306             """
2307             with lock:
2308                 refetch_manifest(format_id, delay)
2309
2310             f = next((f for f in formats if f['format_id'] == format_id), None)
2311             if not f:
2312                 if not is_live:
2313                     self.to_screen(f'{video_id}: Video is no longer live')
2314                 else:
2315                     self.report_warning(
2316                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2317                 return None
2318             return f['manifest_url'], f['manifest_stream_number'], is_live
2319
2320         for f in formats:
2321             f['is_live'] = True
2322             f['protocol'] = 'http_dash_segments_generator'
2323             f['fragments'] = functools.partial(
2324                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2325
2326     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2327         FETCH_SPAN, MAX_DURATION = 5, 432000
2328
2329         mpd_url, stream_number, is_live = None, None, True
2330
2331         begin_index = 0
2332         download_start_time = ctx.get('start') or time.time()
2333
2334         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2335         if lack_early_segments:
2336             self.report_warning(bug_reports_message(
2337                 'Starting download from the last 120 hours of the live stream since '
2338                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2339             lack_early_segments = True
2340
2341         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2342         fragments, fragment_base_url = None, None
2343
2344         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2345             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2346             # Obtain from MPD's maximum seq value
2347             old_mpd_url = mpd_url
2348             last_error = ctx.pop('last_error', None)
2349             expire_fast = immediate or last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2350             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2351                                                or (mpd_url, stream_number, False))
2352             if not refresh_sequence:
2353                 if expire_fast and not is_live:
2354                     return False, last_seq
2355                 elif old_mpd_url == mpd_url:
2356                     return True, last_seq
2357             try:
2358                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2359                     mpd_url, None, note=False, errnote=False, fatal=False)
2360             except ExtractorError:
2361                 fmts = None
2362             if not fmts:
2363                 no_fragment_score += 2
2364                 return False, last_seq
2365             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2366             fragments = fmt_info['fragments']
2367             fragment_base_url = fmt_info['fragment_base_url']
2368             assert fragment_base_url
2369
2370             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2371             return True, _last_seq
2372
2373         while is_live:
2374             fetch_time = time.time()
2375             if no_fragment_score > 30:
2376                 return
2377             if last_segment_url:
2378                 # Obtain from "X-Head-Seqnum" header value from each segment
2379                 try:
2380                     urlh = self._request_webpage(
2381                         last_segment_url, None, note=False, errnote=False, fatal=False)
2382                 except ExtractorError:
2383                     urlh = None
2384                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2385                 if last_seq is None:
2386                     no_fragment_score += 2
2387                     last_segment_url = None
2388                     continue
2389             else:
2390                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2391                 no_fragment_score += 2
2392                 if not should_continue:
2393                     continue
2394
2395             if known_idx > last_seq:
2396                 last_segment_url = None
2397                 continue
2398
2399             last_seq += 1
2400
2401             if begin_index < 0 and known_idx < 0:
2402                 # skip from the start when it's negative value
2403                 known_idx = last_seq + begin_index
2404             if lack_early_segments:
2405                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2406             try:
2407                 for idx in range(known_idx, last_seq):
2408                     # do not update sequence here or you'll get skipped some part of it
2409                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2410                     if not should_continue:
2411                         known_idx = idx - 1
2412                         raise ExtractorError('breaking out of outer loop')
2413                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2414                     yield {
2415                         'url': last_segment_url,
2416                         'fragment_count': last_seq,
2417                     }
2418                 if known_idx == last_seq:
2419                     no_fragment_score += 5
2420                 else:
2421                     no_fragment_score = 0
2422                 known_idx = last_seq
2423             except ExtractorError:
2424                 continue
2425
2426             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2427
2428     def _extract_player_url(self, *ytcfgs, webpage=None):
2429         player_url = traverse_obj(
2430             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2431             get_all=False, expected_type=compat_str)
2432         if not player_url:
2433             return
2434         return urljoin('https://www.youtube.com', player_url)
2435
2436     def _download_player_url(self, video_id, fatal=False):
2437         res = self._download_webpage(
2438             'https://www.youtube.com/iframe_api',
2439             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2440         if res:
2441             player_version = self._search_regex(
2442                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2443             if player_version:
2444                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2445
2446     def _signature_cache_id(self, example_sig):
2447         """ Return a string representation of a signature """
2448         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2449
2450     @classmethod
2451     def _extract_player_info(cls, player_url):
2452         for player_re in cls._PLAYER_INFO_RE:
2453             id_m = re.search(player_re, player_url)
2454             if id_m:
2455                 break
2456         else:
2457             raise ExtractorError('Cannot identify player %r' % player_url)
2458         return id_m.group('id')
2459
2460     def _load_player(self, video_id, player_url, fatal=True):
2461         player_id = self._extract_player_info(player_url)
2462         if player_id not in self._code_cache:
2463             code = self._download_webpage(
2464                 player_url, video_id, fatal=fatal,
2465                 note='Downloading player ' + player_id,
2466                 errnote='Download of %s failed' % player_url)
2467             if code:
2468                 self._code_cache[player_id] = code
2469         return self._code_cache.get(player_id)
2470
2471     def _extract_signature_function(self, video_id, player_url, example_sig):
2472         player_id = self._extract_player_info(player_url)
2473
2474         # Read from filesystem cache
2475         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2476         assert os.path.basename(func_id) == func_id
2477
2478         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
2479         if cache_spec is not None:
2480             return lambda s: ''.join(s[i] for i in cache_spec)
2481
2482         code = self._load_player(video_id, player_url)
2483         if code:
2484             res = self._parse_sig_js(code)
2485
2486             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2487             cache_res = res(test_string)
2488             cache_spec = [ord(c) for c in cache_res]
2489
2490             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
2491             return res
2492
2493     def _print_sig_code(self, func, example_sig):
2494         if not self.get_param('youtube_print_sig_code'):
2495             return
2496
2497         def gen_sig_code(idxs):
2498             def _genslice(start, end, step):
2499                 starts = '' if start == 0 else str(start)
2500                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2501                 steps = '' if step == 1 else (':%d' % step)
2502                 return f's[{starts}{ends}{steps}]'
2503
2504             step = None
2505             # Quelch pyflakes warnings - start will be set when step is set
2506             start = '(Never used)'
2507             for i, prev in zip(idxs[1:], idxs[:-1]):
2508                 if step is not None:
2509                     if i - prev == step:
2510                         continue
2511                     yield _genslice(start, prev, step)
2512                     step = None
2513                     continue
2514                 if i - prev in [-1, 1]:
2515                     step = i - prev
2516                     start = prev
2517                     continue
2518                 else:
2519                     yield 's[%d]' % prev
2520             if step is None:
2521                 yield 's[%d]' % i
2522             else:
2523                 yield _genslice(start, i, step)
2524
2525         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2526         cache_res = func(test_string)
2527         cache_spec = [ord(c) for c in cache_res]
2528         expr_code = ' + '.join(gen_sig_code(cache_spec))
2529         signature_id_tuple = '(%s)' % (
2530             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2531         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2532                 '    return %s\n') % (signature_id_tuple, expr_code)
2533         self.to_screen('Extracted signature function:\n' + code)
2534
2535     def _parse_sig_js(self, jscode):
2536         funcname = self._search_regex(
2537             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2538              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2539              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2540              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2541              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2542              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2543              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2544              # Obsolete patterns
2545              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2546              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2547              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2548              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2549              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2550              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2551              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2552              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2553             jscode, 'Initial JS player signature function name', group='sig')
2554
2555         jsi = JSInterpreter(jscode)
2556         initial_function = jsi.extract_function(funcname)
2557         return lambda s: initial_function([s])
2558
2559     def _decrypt_signature(self, s, video_id, player_url):
2560         """Turn the encrypted s field into a working signature"""
2561         try:
2562             player_id = (player_url, self._signature_cache_id(s))
2563             if player_id not in self._player_cache:
2564                 func = self._extract_signature_function(video_id, player_url, s)
2565                 self._player_cache[player_id] = func
2566             func = self._player_cache[player_id]
2567             self._print_sig_code(func, s)
2568             return func(s)
2569         except Exception as e:
2570             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2571
2572     def _decrypt_nsig(self, s, video_id, player_url):
2573         """Turn the encrypted n field into a working signature"""
2574         if player_url is None:
2575             raise ExtractorError('Cannot decrypt nsig without player_url')
2576         player_url = urljoin('https://www.youtube.com', player_url)
2577
2578         sig_id = ('nsig_value', s)
2579         if sig_id in self._player_cache:
2580             return self._player_cache[sig_id]
2581
2582         try:
2583             player_id = ('nsig', player_url)
2584             if player_id not in self._player_cache:
2585                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2586             func = self._player_cache[player_id]
2587             self._player_cache[sig_id] = func(s)
2588             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2589             return self._player_cache[sig_id]
2590         except Exception as e:
2591             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2592
2593     def _extract_n_function_name(self, jscode):
2594         nfunc, idx = self._search_regex(
2595             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2596             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2597         if not idx:
2598             return nfunc
2599         return json.loads(js_to_json(self._search_regex(
2600             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2601             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2602
2603     def _extract_n_function(self, video_id, player_url):
2604         player_id = self._extract_player_info(player_url)
2605         func_code = self._downloader.cache.load('youtube-nsig', player_id)
2606
2607         if func_code:
2608             jsi = JSInterpreter(func_code)
2609         else:
2610             jscode = self._load_player(video_id, player_url)
2611             funcname = self._extract_n_function_name(jscode)
2612             jsi = JSInterpreter(jscode)
2613             func_code = jsi.extract_function_code(funcname)
2614             self._downloader.cache.store('youtube-nsig', player_id, func_code)
2615
2616         if self.get_param('youtube_print_sig_code'):
2617             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2618
2619         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2620
2621     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2622         """
2623         Extract signatureTimestamp (sts)
2624         Required to tell API what sig/player version is in use.
2625         """
2626         sts = None
2627         if isinstance(ytcfg, dict):
2628             sts = int_or_none(ytcfg.get('STS'))
2629
2630         if not sts:
2631             # Attempt to extract from player
2632             if player_url is None:
2633                 error_msg = 'Cannot extract signature timestamp without player_url.'
2634                 if fatal:
2635                     raise ExtractorError(error_msg)
2636                 self.report_warning(error_msg)
2637                 return
2638             code = self._load_player(video_id, player_url, fatal=fatal)
2639             if code:
2640                 sts = int_or_none(self._search_regex(
2641                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2642                     'JS player signature timestamp', group='sts', fatal=fatal))
2643         return sts
2644
2645     def _mark_watched(self, video_id, player_responses):
2646         playback_url = get_first(
2647             player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2648             expected_type=url_or_none)
2649         if not playback_url:
2650             self.report_warning('Unable to mark watched')
2651             return
2652         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2653         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2654
2655         # cpn generation algorithm is reverse engineered from base.js.
2656         # In fact it works even with dummy cpn.
2657         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2658         cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2659
2660         qs.update({
2661             'ver': ['2'],
2662             'cpn': [cpn],
2663         })
2664         playback_url = compat_urlparse.urlunparse(
2665             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2666
2667         self._download_webpage(
2668             playback_url, video_id, 'Marking watched',
2669             'Unable to mark watched', fatal=False)
2670
2671     @staticmethod
2672     def _extract_urls(webpage):
2673         # Embedded YouTube player
2674         entries = [
2675             unescapeHTML(mobj.group('url'))
2676             for mobj in re.finditer(r'''(?x)
2677             (?:
2678                 <iframe[^>]+?src=|
2679                 data-video-url=|
2680                 <embed[^>]+?src=|
2681                 embedSWF\(?:\s*|
2682                 <object[^>]+data=|
2683                 new\s+SWFObject\(
2684             )
2685             (["\'])
2686                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2687                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2688             \1''', webpage)]
2689
2690         # lazyYT YouTube embed
2691         entries.extend(list(map(
2692             unescapeHTML,
2693             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2694
2695         # Wordpress "YouTube Video Importer" plugin
2696         matches = re.findall(r'''(?x)<div[^>]+
2697             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2698             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2699         entries.extend(m[-1] for m in matches)
2700
2701         return entries
2702
2703     @staticmethod
2704     def _extract_url(webpage):
2705         urls = YoutubeIE._extract_urls(webpage)
2706         return urls[0] if urls else None
2707
2708     @classmethod
2709     def extract_id(cls, url):
2710         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2711         if mobj is None:
2712             raise ExtractorError('Invalid URL: %s' % url)
2713         return mobj.group('id')
2714
2715     def _extract_chapters_from_json(self, data, duration):
2716         chapter_list = traverse_obj(
2717             data, (
2718                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2719                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2720             ), expected_type=list)
2721
2722         return self._extract_chapters(
2723             chapter_list,
2724             chapter_time=lambda chapter: float_or_none(
2725                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2726             chapter_title=lambda chapter: traverse_obj(
2727                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2728             duration=duration)
2729
2730     def _extract_chapters_from_engagement_panel(self, data, duration):
2731         content_list = traverse_obj(
2732             data,
2733             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2734             expected_type=list, default=[])
2735         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2736         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2737
2738         return next(filter(None, (
2739             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2740                                    chapter_time, chapter_title, duration)
2741             for contents in content_list)), [])
2742
2743     def _extract_chapters_from_description(self, description, duration):
2744         return self._extract_chapters(
2745             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
2746             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
2747             duration=duration, strict=False)
2748
2749     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
2750         if not duration:
2751             return
2752         chapter_list = [{
2753             'start_time': chapter_time(chapter),
2754             'title': chapter_title(chapter),
2755         } for chapter in chapter_list or []]
2756         if not strict:
2757             chapter_list.sort(key=lambda c: c['start_time'] or 0)
2758
2759         chapters = [{'start_time': 0, 'title': '<Untitled>'}]
2760         for idx, chapter in enumerate(chapter_list):
2761             if chapter['start_time'] is None or not chapter['title']:
2762                 self.report_warning(f'Incomplete chapter {idx}')
2763             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
2764                 chapters[-1]['end_time'] = chapter['start_time']
2765                 chapters.append(chapter)
2766             else:
2767                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
2768         chapters[-1]['end_time'] = duration
2769         return chapters if len(chapters) > 1 and chapters[1]['start_time'] else chapters[1:]
2770
2771     def _extract_comment(self, comment_renderer, parent=None):
2772         comment_id = comment_renderer.get('commentId')
2773         if not comment_id:
2774             return
2775
2776         text = self._get_text(comment_renderer, 'contentText')
2777
2778         # note: timestamp is an estimate calculated from the current time and time_text
2779         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2780         author = self._get_text(comment_renderer, 'authorText')
2781         author_id = try_get(comment_renderer,
2782                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2783
2784         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2785                                                        lambda x: x['likeCount']), compat_str)) or 0
2786         author_thumbnail = try_get(comment_renderer,
2787                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2788
2789         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2790         is_favorited = 'creatorHeart' in (try_get(
2791             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2792         return {
2793             'id': comment_id,
2794             'text': text,
2795             'timestamp': timestamp,
2796             'time_text': time_text,
2797             'like_count': votes,
2798             'is_favorited': is_favorited,
2799             'author': author,
2800             'author_id': author_id,
2801             'author_thumbnail': author_thumbnail,
2802             'author_is_uploader': author_is_uploader,
2803             'parent': parent or 'root'
2804         }
2805
2806     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2807
2808         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2809
2810         def extract_header(contents):
2811             _continuation = None
2812             for content in contents:
2813                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2814                 expected_comment_count = self._get_count(
2815                     comments_header_renderer, 'countText', 'commentsCount')
2816
2817                 if expected_comment_count:
2818                     tracker['est_total'] = expected_comment_count
2819                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2820                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2821
2822                 sort_menu_item = try_get(
2823                     comments_header_renderer,
2824                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2825                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2826
2827                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2828                 if not _continuation:
2829                     continue
2830
2831                 sort_text = str_or_none(sort_menu_item.get('title'))
2832                 if not sort_text:
2833                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2834                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2835                 break
2836             return _continuation
2837
2838         def extract_thread(contents):
2839             if not parent:
2840                 tracker['current_page_thread'] = 0
2841             for content in contents:
2842                 if not parent and tracker['total_parent_comments'] >= max_parents:
2843                     yield
2844                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2845                 comment_renderer = get_first(
2846                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2847                     expected_type=dict, default={})
2848
2849                 comment = self._extract_comment(comment_renderer, parent)
2850                 if not comment:
2851                     continue
2852
2853                 tracker['running_total'] += 1
2854                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2855                 yield comment
2856
2857                 # Attempt to get the replies
2858                 comment_replies_renderer = try_get(
2859                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2860
2861                 if comment_replies_renderer:
2862                     tracker['current_page_thread'] += 1
2863                     comment_entries_iter = self._comment_entries(
2864                         comment_replies_renderer, ytcfg, video_id,
2865                         parent=comment.get('id'), tracker=tracker)
2866                     yield from itertools.islice(comment_entries_iter, min(
2867                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2868
2869         # Keeps track of counts across recursive calls
2870         if not tracker:
2871             tracker = dict(
2872                 running_total=0,
2873                 est_total=0,
2874                 current_page_thread=0,
2875                 total_parent_comments=0,
2876                 total_reply_comments=0)
2877
2878         # TODO: Deprecated
2879         # YouTube comments have a max depth of 2
2880         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2881         if max_depth:
2882             self._downloader.deprecation_warning(
2883                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2884         if max_depth == 1 and parent:
2885             return
2886
2887         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2888             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2889
2890         continuation = self._extract_continuation(root_continuation_data)
2891
2892         response = None
2893         is_forced_continuation = False
2894         is_first_continuation = parent is None
2895         if is_first_continuation and not continuation:
2896             # Sometimes you can get comments by generating the continuation yourself,
2897             # even if YouTube initially reports them being disabled - e.g. stories comments.
2898             # Note: if the comment section is actually disabled, YouTube may return a response with
2899             # required check_get_keys missing. So we will disable that check initially in this case.
2900             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2901             is_forced_continuation = True
2902
2903         for page_num in itertools.count(0):
2904             if not continuation:
2905                 break
2906             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2907             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2908             if page_num == 0:
2909                 if is_first_continuation:
2910                     note_prefix = 'Downloading comment section API JSON'
2911                 else:
2912                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2913                         tracker['current_page_thread'], comment_prog_str)
2914             else:
2915                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2916                     '       ' if parent else '', ' replies' if parent else '',
2917                     page_num, comment_prog_str)
2918
2919             response = self._extract_response(
2920                 item_id=None, query=continuation,
2921                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2922                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2923             is_forced_continuation = False
2924             continuation_contents = traverse_obj(
2925                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2926
2927             continuation = None
2928             for continuation_section in continuation_contents:
2929                 continuation_items = traverse_obj(
2930                     continuation_section,
2931                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2932                     get_all=False, expected_type=list) or []
2933                 if is_first_continuation:
2934                     continuation = extract_header(continuation_items)
2935                     is_first_continuation = False
2936                     if continuation:
2937                         break
2938                     continue
2939
2940                 for entry in extract_thread(continuation_items):
2941                     if not entry:
2942                         return
2943                     yield entry
2944                 continuation = self._extract_continuation({'contents': continuation_items})
2945                 if continuation:
2946                     break
2947
2948         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2949         if message and not parent and tracker['running_total'] == 0:
2950             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
2951
2952     @staticmethod
2953     def _generate_comment_continuation(video_id):
2954         """
2955         Generates initial comment section continuation token from given video id
2956         """
2957         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
2958         return base64.b64encode(token.encode()).decode()
2959
2960     def _get_comments(self, ytcfg, video_id, contents, webpage):
2961         """Entry for comment extraction"""
2962         def _real_comment_extract(contents):
2963             renderer = next((
2964                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2965                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2966             yield from self._comment_entries(renderer, ytcfg, video_id)
2967
2968         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2969         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2970
2971     @staticmethod
2972     def _get_checkok_params():
2973         return {'contentCheckOk': True, 'racyCheckOk': True}
2974
2975     @classmethod
2976     def _generate_player_context(cls, sts=None):
2977         context = {
2978             'html5Preference': 'HTML5_PREF_WANTS',
2979         }
2980         if sts is not None:
2981             context['signatureTimestamp'] = sts
2982         return {
2983             'playbackContext': {
2984                 'contentPlaybackContext': context
2985             },
2986             **cls._get_checkok_params()
2987         }
2988
2989     @staticmethod
2990     def _is_agegated(player_response):
2991         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2992             return True
2993
2994         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2995         AGE_GATE_REASONS = (
2996             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2997             'age_verification_required', 'age_check_required',  # status
2998         )
2999         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3000
3001     @staticmethod
3002     def _is_unplayable(player_response):
3003         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3004
3005     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
3006
3007         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3008         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3009         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3010         headers = self.generate_api_headers(
3011             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3012
3013         yt_query = {
3014             'videoId': video_id,
3015             'params': '8AEB'  # enable stories
3016         }
3017         yt_query.update(self._generate_player_context(sts))
3018         return self._extract_response(
3019             item_id=video_id, ep='player', query=yt_query,
3020             ytcfg=player_ytcfg, headers=headers, fatal=True,
3021             default_client=client,
3022             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3023         ) or None
3024
3025     def _get_requested_clients(self, url, smuggled_data):
3026         requested_clients = []
3027         default = ['android', 'web']
3028         allowed_clients = sorted(
3029             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3030             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3031         for client in self._configuration_arg('player_client'):
3032             if client in allowed_clients:
3033                 requested_clients.append(client)
3034             elif client == 'default':
3035                 requested_clients.extend(default)
3036             elif client == 'all':
3037                 requested_clients.extend(allowed_clients)
3038             else:
3039                 self.report_warning(f'Skipping unsupported client {client}')
3040         if not requested_clients:
3041             requested_clients = default
3042
3043         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3044             requested_clients.extend(
3045                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3046
3047         return orderedSet(requested_clients)
3048
3049     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3050         initial_pr = None
3051         if webpage:
3052             initial_pr = self._search_json(
3053                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3054
3055         all_clients = set(clients)
3056         clients = clients[::-1]
3057         prs = []
3058
3059         def append_client(*client_names):
3060             """ Append the first client name that exists but not already used """
3061             for client_name in client_names:
3062                 actual_client = _split_innertube_client(client_name)[0]
3063                 if actual_client in INNERTUBE_CLIENTS:
3064                     if actual_client not in all_clients:
3065                         clients.append(client_name)
3066                         all_clients.add(actual_client)
3067                         return
3068
3069         # Android player_response does not have microFormats which are needed for
3070         # extraction of some data. So we return the initial_pr with formats
3071         # stripped out even if not requested by the user
3072         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3073         if initial_pr:
3074             pr = dict(initial_pr)
3075             pr['streamingData'] = None
3076             prs.append(pr)
3077
3078         last_error = None
3079         tried_iframe_fallback = False
3080         player_url = None
3081         while clients:
3082             client, base_client, variant = _split_innertube_client(clients.pop())
3083             player_ytcfg = master_ytcfg if client == 'web' else {}
3084             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3085                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3086
3087             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3088             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3089             if 'js' in self._configuration_arg('player_skip'):
3090                 require_js_player = False
3091                 player_url = None
3092
3093             if not player_url and not tried_iframe_fallback and require_js_player:
3094                 player_url = self._download_player_url(video_id)
3095                 tried_iframe_fallback = True
3096
3097             try:
3098                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3099                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3100             except ExtractorError as e:
3101                 if last_error:
3102                     self.report_warning(last_error)
3103                 last_error = e
3104                 continue
3105
3106             if pr:
3107                 prs.append(pr)
3108
3109             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3110             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3111                 append_client(f'{base_client}_creator')
3112             elif self._is_agegated(pr):
3113                 if variant == 'tv_embedded':
3114                     append_client(f'{base_client}_embedded')
3115                 elif not variant:
3116                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3117
3118         if last_error:
3119             if not len(prs):
3120                 raise last_error
3121             self.report_warning(last_error)
3122         return prs, player_url
3123
3124     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3125         itags, stream_ids = {}, []
3126         itag_qualities, res_qualities = {}, {}
3127         q = qualities([
3128             # Normally tiny is the smallest video-only formats. But
3129             # audio-only formats with unknown quality may get tagged as tiny
3130             'tiny',
3131             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3132             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3133         ])
3134         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3135
3136         for fmt in streaming_formats:
3137             if fmt.get('targetDurationSec'):
3138                 continue
3139
3140             itag = str_or_none(fmt.get('itag'))
3141             audio_track = fmt.get('audioTrack') or {}
3142             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3143             if stream_id in stream_ids:
3144                 continue
3145
3146             quality = fmt.get('quality')
3147             height = int_or_none(fmt.get('height'))
3148             if quality == 'tiny' or not quality:
3149                 quality = fmt.get('audioQuality', '').lower() or quality
3150             # The 3gp format (17) in android client has a quality of "small",
3151             # but is actually worse than other formats
3152             if itag == '17':
3153                 quality = 'tiny'
3154             if quality:
3155                 if itag:
3156                     itag_qualities[itag] = quality
3157                 if height:
3158                     res_qualities[height] = quality
3159             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3160             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3161             # number of fragment that would subsequently requested with (`&sq=N`)
3162             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3163                 continue
3164
3165             fmt_url = fmt.get('url')
3166             if not fmt_url:
3167                 sc = compat_parse_qs(fmt.get('signatureCipher'))
3168                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3169                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3170                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3171                     continue
3172                 try:
3173                     fmt_url += '&%s=%s' % (
3174                         traverse_obj(sc, ('sp', -1)) or 'signature',
3175                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3176                     )
3177                 except ExtractorError as e:
3178                     self.report_warning('Signature extraction failed: Some formats may be missing', only_once=True)
3179                     self.write_debug(e, only_once=True)
3180                     continue
3181
3182             query = parse_qs(fmt_url)
3183             throttled = False
3184             if query.get('n'):
3185                 try:
3186                     fmt_url = update_url_query(fmt_url, {
3187                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3188                 except ExtractorError as e:
3189                     self.report_warning(
3190                         'nsig extraction failed: You may experience throttling for some formats\n'
3191                         f'n = {query["n"][0]} ; player = {player_url}', only_once=True)
3192                     self.write_debug(e, only_once=True)
3193                     throttled = True
3194
3195             if itag:
3196                 itags[itag] = 'https'
3197                 stream_ids.append(stream_id)
3198
3199             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3200             language_preference = (
3201                 10 if audio_track.get('audioIsDefault') and 10
3202                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3203                 else -1)
3204             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3205             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3206             # Make sure to avoid false positives with small duration differences.
3207             # Eg: __2ABJjxzNo, ySuUZEjARPY
3208             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3209             if is_damaged:
3210                 self.report_warning(
3211                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3212             dct = {
3213                 'asr': int_or_none(fmt.get('audioSampleRate')),
3214                 'filesize': int_or_none(fmt.get('contentLength')),
3215                 'format_id': itag,
3216                 'format_note': join_nonempty(
3217                     '%s%s' % (audio_track.get('displayName') or '',
3218                               ' (default)' if language_preference > 0 else ''),
3219                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3220                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3221                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3222                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3223                 'fps': int_or_none(fmt.get('fps')) or None,
3224                 'height': height,
3225                 'quality': q(quality),
3226                 'has_drm': bool(fmt.get('drmFamilies')),
3227                 'tbr': tbr,
3228                 'url': fmt_url,
3229                 'width': int_or_none(fmt.get('width')),
3230                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3231                                           'desc' if language_preference < -1 else ''),
3232                 'language_preference': language_preference,
3233                 # Strictly de-prioritize damaged and 3gp formats
3234                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3235             }
3236             mime_mobj = re.match(
3237                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3238             if mime_mobj:
3239                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3240                 dct.update(parse_codecs(mime_mobj.group(2)))
3241             no_audio = dct.get('acodec') == 'none'
3242             no_video = dct.get('vcodec') == 'none'
3243             if no_audio:
3244                 dct['vbr'] = tbr
3245             if no_video:
3246                 dct['abr'] = tbr
3247             if no_audio or no_video:
3248                 dct['downloader_options'] = {
3249                     # Youtube throttles chunks >~10M
3250                     'http_chunk_size': 10485760,
3251                 }
3252                 if dct.get('ext'):
3253                     dct['container'] = dct['ext'] + '_dash'
3254             yield dct
3255
3256         live_from_start = is_live and self.get_param('live_from_start')
3257         skip_manifests = self._configuration_arg('skip')
3258         if not self.get_param('youtube_include_hls_manifest', True):
3259             skip_manifests.append('hls')
3260         if not self.get_param('youtube_include_dash_manifest', True):
3261             skip_manifests.append('dash')
3262         get_dash = 'dash' not in skip_manifests and (
3263             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3264         get_hls = not live_from_start and 'hls' not in skip_manifests
3265
3266         def process_manifest_format(f, proto, itag):
3267             if itag in itags:
3268                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3269                     return False
3270                 itag = f'{itag}-{proto}'
3271             if itag:
3272                 f['format_id'] = itag
3273                 itags[itag] = proto
3274
3275             f['quality'] = next((
3276                 q(qdict[val])
3277                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3278                 if val in qdict), -1)
3279             return True
3280
3281         for sd in streaming_data:
3282             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3283             if hls_manifest_url:
3284                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3285                     if process_manifest_format(f, 'hls', self._search_regex(
3286                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3287                         yield f
3288
3289             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3290             if dash_manifest_url:
3291                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3292                     if process_manifest_format(f, 'dash', f['format_id']):
3293                         f['filesize'] = int_or_none(self._search_regex(
3294                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3295                         if live_from_start:
3296                             f['is_from_start'] = True
3297
3298                         yield f
3299
3300     def _extract_storyboard(self, player_responses, duration):
3301         spec = get_first(
3302             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3303         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3304         if not base_url:
3305             return
3306         L = len(spec) - 1
3307         for i, args in enumerate(spec):
3308             args = args.split('#')
3309             counts = list(map(int_or_none, args[:5]))
3310             if len(args) != 8 or not all(counts):
3311                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3312                 continue
3313             width, height, frame_count, cols, rows = counts
3314             N, sigh = args[6:]
3315
3316             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3317             fragment_count = frame_count / (cols * rows)
3318             fragment_duration = duration / fragment_count
3319             yield {
3320                 'format_id': f'sb{i}',
3321                 'format_note': 'storyboard',
3322                 'ext': 'mhtml',
3323                 'protocol': 'mhtml',
3324                 'acodec': 'none',
3325                 'vcodec': 'none',
3326                 'url': url,
3327                 'width': width,
3328                 'height': height,
3329                 'fragments': [{
3330                     'url': url.replace('$M', str(j)),
3331                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3332                 } for j in range(math.ceil(fragment_count))],
3333             }
3334
3335     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3336         webpage = None
3337         if 'webpage' not in self._configuration_arg('player_skip'):
3338             webpage = self._download_webpage(
3339                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3340
3341         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3342
3343         player_responses, player_url = self._extract_player_responses(
3344             self._get_requested_clients(url, smuggled_data),
3345             video_id, webpage, master_ytcfg)
3346
3347         return webpage, master_ytcfg, player_responses, player_url
3348
3349     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3350         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3351         is_live = get_first(video_details, 'isLive')
3352         if is_live is None:
3353             is_live = get_first(live_broadcast_details, 'isLiveNow')
3354
3355         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3356         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3357
3358         return live_broadcast_details, is_live, streaming_data, formats
3359
3360     def _real_extract(self, url):
3361         url, smuggled_data = unsmuggle_url(url, {})
3362         video_id = self._match_id(url)
3363
3364         base_url = self.http_scheme() + '//www.youtube.com/'
3365         webpage_url = base_url + 'watch?v=' + video_id
3366
3367         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3368
3369         playability_statuses = traverse_obj(
3370             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3371
3372         trailer_video_id = get_first(
3373             playability_statuses,
3374             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3375             expected_type=str)
3376         if trailer_video_id:
3377             return self.url_result(
3378                 trailer_video_id, self.ie_key(), trailer_video_id)
3379
3380         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3381                        if webpage else (lambda x: None))
3382
3383         video_details = traverse_obj(
3384             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3385         microformats = traverse_obj(
3386             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3387             expected_type=dict, default=[])
3388         video_title = (
3389             get_first(video_details, 'title')
3390             or self._get_text(microformats, (..., 'title'))
3391             or search_meta(['og:title', 'twitter:title', 'title']))
3392         video_description = get_first(video_details, 'shortDescription')
3393
3394         multifeed_metadata_list = get_first(
3395             player_responses,
3396             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3397             expected_type=str)
3398         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3399             if self.get_param('noplaylist'):
3400                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3401             else:
3402                 entries = []
3403                 feed_ids = []
3404                 for feed in multifeed_metadata_list.split(','):
3405                     # Unquote should take place before split on comma (,) since textual
3406                     # fields may contain comma as well (see
3407                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3408                     feed_data = compat_parse_qs(
3409                         compat_urllib_parse_unquote_plus(feed))
3410
3411                     def feed_entry(name):
3412                         return try_get(
3413                             feed_data, lambda x: x[name][0], compat_str)
3414
3415                     feed_id = feed_entry('id')
3416                     if not feed_id:
3417                         continue
3418                     feed_title = feed_entry('title')
3419                     title = video_title
3420                     if feed_title:
3421                         title += ' (%s)' % feed_title
3422                     entries.append({
3423                         '_type': 'url_transparent',
3424                         'ie_key': 'Youtube',
3425                         'url': smuggle_url(
3426                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3427                             {'force_singlefeed': True}),
3428                         'title': title,
3429                     })
3430                     feed_ids.append(feed_id)
3431                 self.to_screen(
3432                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3433                     % (', '.join(feed_ids), video_id))
3434                 return self.playlist_result(
3435                     entries, video_id, video_title, video_description)
3436
3437         duration = int_or_none(
3438             get_first(video_details, 'lengthSeconds')
3439             or get_first(microformats, 'lengthSeconds')
3440             or parse_duration(search_meta('duration'))) or None
3441
3442         if get_first(video_details, 'isPostLiveDvr'):
3443             self.write_debug('Video is in Post-Live Manifestless mode')
3444             if duration or 0 > 4 * 3600:
3445                 self.report_warning(
3446                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3447                     'This is a known issue and patches are welcome')
3448
3449         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3450             video_id, microformats, video_details, player_responses, player_url, duration)
3451
3452         if not formats:
3453             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3454                 self.report_drm(video_id)
3455             pemr = get_first(
3456                 playability_statuses,
3457                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3458             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3459             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3460             if subreason:
3461                 if subreason == 'The uploader has not made this video available in your country.':
3462                     countries = get_first(microformats, 'availableCountries')
3463                     if not countries:
3464                         regions_allowed = search_meta('regionsAllowed')
3465                         countries = regions_allowed.split(',') if regions_allowed else None
3466                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3467                 reason += f'. {subreason}'
3468             if reason:
3469                 self.raise_no_formats(reason, expected=True)
3470
3471         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3472         if not keywords and webpage:
3473             keywords = [
3474                 unescapeHTML(m.group('content'))
3475                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3476         for keyword in keywords:
3477             if keyword.startswith('yt:stretch='):
3478                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3479                 if mobj:
3480                     # NB: float is intentional for forcing float division
3481                     w, h = (float(v) for v in mobj.groups())
3482                     if w > 0 and h > 0:
3483                         ratio = w / h
3484                         for f in formats:
3485                             if f.get('vcodec') != 'none':
3486                                 f['stretched_ratio'] = ratio
3487                         break
3488         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3489         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3490         if thumbnail_url:
3491             thumbnails.append({
3492                 'url': thumbnail_url,
3493             })
3494         original_thumbnails = thumbnails.copy()
3495
3496         # The best resolution thumbnails sometimes does not appear in the webpage
3497         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3498         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3499         thumbnail_names = [
3500             # While the *1,*2,*3 thumbnails are just below their correspnding "*default" variants
3501             # in resolution, these are not the custom thumbnail. So de-prioritize them
3502             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3503             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3504         ]
3505         n_thumbnail_names = len(thumbnail_names)
3506         thumbnails.extend({
3507             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3508                 video_id=video_id, name=name, ext=ext,
3509                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3510         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3511         for thumb in thumbnails:
3512             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3513             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3514         self._remove_duplicate_formats(thumbnails)
3515         self._downloader._sort_thumbnails(original_thumbnails)
3516
3517         category = get_first(microformats, 'category') or search_meta('genre')
3518         channel_id = str_or_none(
3519             get_first(video_details, 'channelId')
3520             or get_first(microformats, 'externalChannelId')
3521             or search_meta('channelId'))
3522         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3523
3524         live_content = get_first(video_details, 'isLiveContent')
3525         is_upcoming = get_first(video_details, 'isUpcoming')
3526         if is_live is None:
3527             if is_upcoming or live_content is False:
3528                 is_live = False
3529         if is_upcoming is None and (live_content or is_live):
3530             is_upcoming = False
3531         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3532         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3533         if not duration and live_end_time and live_start_time:
3534             duration = live_end_time - live_start_time
3535
3536         if is_live and self.get_param('live_from_start'):
3537             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3538
3539         formats.extend(self._extract_storyboard(player_responses, duration))
3540
3541         # Source is given priority since formats that throttle are given lower source_preference
3542         # When throttling issue is fully fixed, remove this
3543         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3544
3545         info = {
3546             'id': video_id,
3547             'title': video_title,
3548             'formats': formats,
3549             'thumbnails': thumbnails,
3550             # The best thumbnail that we are sure exists. Prevents unnecessary
3551             # URL checking if user don't care about getting the best possible thumbnail
3552             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3553             'description': video_description,
3554             'uploader': get_first(video_details, 'author'),
3555             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3556             'uploader_url': owner_profile_url,
3557             'channel_id': channel_id,
3558             'channel_url': format_field(channel_id, template='https://www.youtube.com/channel/%s'),
3559             'duration': duration,
3560             'view_count': int_or_none(
3561                 get_first((video_details, microformats), (..., 'viewCount'))
3562                 or search_meta('interactionCount')),
3563             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3564             'age_limit': 18 if (
3565                 get_first(microformats, 'isFamilySafe') is False
3566                 or search_meta('isFamilyFriendly') == 'false'
3567                 or search_meta('og:restrictions:age') == '18+') else 0,
3568             'webpage_url': webpage_url,
3569             'categories': [category] if category else None,
3570             'tags': keywords,
3571             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3572             'is_live': is_live,
3573             'was_live': (False if is_live or is_upcoming or live_content is False
3574                          else None if is_live is None or is_upcoming is None
3575                          else live_content),
3576             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3577             'release_timestamp': live_start_time,
3578         }
3579
3580         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3581         if pctr:
3582             def get_lang_code(track):
3583                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3584                         or track.get('languageCode'))
3585
3586             # Converted into dicts to remove duplicates
3587             captions = {
3588                 get_lang_code(sub): sub
3589                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3590             translation_languages = {
3591                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3592                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3593
3594             def process_language(container, base_url, lang_code, sub_name, query):
3595                 lang_subs = container.setdefault(lang_code, [])
3596                 for fmt in self._SUBTITLE_FORMATS:
3597                     query.update({
3598                         'fmt': fmt,
3599                     })
3600                     lang_subs.append({
3601                         'ext': fmt,
3602                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3603                         'name': sub_name,
3604                     })
3605
3606             subtitles, automatic_captions = {}, {}
3607             for lang_code, caption_track in captions.items():
3608                 base_url = caption_track.get('baseUrl')
3609                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3610                 if not base_url:
3611                     continue
3612                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3613                 if caption_track.get('kind') != 'asr':
3614                     if not lang_code:
3615                         continue
3616                     process_language(
3617                         subtitles, base_url, lang_code, lang_name, {})
3618                     if not caption_track.get('isTranslatable'):
3619                         continue
3620                 for trans_code, trans_name in translation_languages.items():
3621                     if not trans_code:
3622                         continue
3623                     orig_trans_code = trans_code
3624                     if caption_track.get('kind') != 'asr':
3625                         if 'translated_subs' in self._configuration_arg('skip'):
3626                             continue
3627                         trans_code += f'-{lang_code}'
3628                         trans_name += format_field(lang_name, template=' from %s')
3629                     # Add an "-orig" label to the original language so that it can be distinguished.
3630                     # The subs are returned without "-orig" as well for compatibility
3631                     if lang_code == f'a-{orig_trans_code}':
3632                         process_language(
3633                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3634                     # Setting tlang=lang returns damaged subtitles.
3635                     process_language(automatic_captions, base_url, trans_code, trans_name,
3636                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3637             info['automatic_captions'] = automatic_captions
3638             info['subtitles'] = subtitles
3639
3640         parsed_url = compat_urllib_parse_urlparse(url)
3641         for component in [parsed_url.fragment, parsed_url.query]:
3642             query = compat_parse_qs(component)
3643             for k, v in query.items():
3644                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3645                     d_k += '_time'
3646                     if d_k not in info and k in s_ks:
3647                         info[d_k] = parse_duration(query[k][0])
3648
3649         # Youtube Music Auto-generated description
3650         if video_description:
3651             mobj = re.search(
3652                 r'''(?xs)
3653                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3654                     (?P<album>[^\n]+)
3655                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3656                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3657                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3658                     .+\nAuto-generated\ by\ YouTube\.\s*$
3659                 ''', video_description)
3660             if mobj:
3661                 release_year = mobj.group('release_year')
3662                 release_date = mobj.group('release_date')
3663                 if release_date:
3664                     release_date = release_date.replace('-', '')
3665                     if not release_year:
3666                         release_year = release_date[:4]
3667                 info.update({
3668                     'album': mobj.group('album'.strip()),
3669                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3670                     'track': mobj.group('track').strip(),
3671                     'release_date': release_date,
3672                     'release_year': int_or_none(release_year),
3673                 })
3674
3675         initial_data = None
3676         if webpage:
3677             initial_data = self._search_json(
3678                 self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', video_id, fatal=False)
3679         if not initial_data:
3680             query = {'videoId': video_id}
3681             query.update(self._get_checkok_params())
3682             initial_data = self._extract_response(
3683                 item_id=video_id, ep='next', fatal=False,
3684                 ytcfg=master_ytcfg, query=query,
3685                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3686                 note='Downloading initial data API JSON')
3687
3688         try:  # This will error if there is no livechat
3689             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3690         except (KeyError, IndexError, TypeError):
3691             pass
3692         else:
3693             info.setdefault('subtitles', {})['live_chat'] = [{
3694                 'url': f'https://www.youtube.com/watch?v={video_id}',  # url is needed to set cookies
3695                 'video_id': video_id,
3696                 'ext': 'json',
3697                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3698             }]
3699
3700         if initial_data:
3701             info['chapters'] = (
3702                 self._extract_chapters_from_json(initial_data, duration)
3703                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3704                 or self._extract_chapters_from_description(video_description, duration)
3705                 or None)
3706
3707         contents = traverse_obj(
3708             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3709             expected_type=list, default=[])
3710
3711         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3712         if vpir:
3713             stl = vpir.get('superTitleLink')
3714             if stl:
3715                 stl = self._get_text(stl)
3716                 if try_get(
3717                         vpir,
3718                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3719                     info['location'] = stl
3720                 else:
3721                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3722                     if mobj:
3723                         info.update({
3724                             'series': mobj.group(1),
3725                             'season_number': int(mobj.group(2)),
3726                             'episode_number': int(mobj.group(3)),
3727                         })
3728             for tlb in (try_get(
3729                     vpir,
3730                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3731                     list) or []):
3732                 tbr = tlb.get('toggleButtonRenderer') or {}
3733                 for getter, regex in [(
3734                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3735                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3736                             lambda x: x['accessibility'],
3737                             lambda x: x['accessibilityData']['accessibilityData'],
3738                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3739                     label = (try_get(tbr, getter, dict) or {}).get('label')
3740                     if label:
3741                         mobj = re.match(regex, label)
3742                         if mobj:
3743                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3744                             break
3745             sbr_tooltip = try_get(
3746                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3747             if sbr_tooltip:
3748                 like_count, dislike_count = sbr_tooltip.split(' / ')
3749                 info.update({
3750                     'like_count': str_to_int(like_count),
3751                     'dislike_count': str_to_int(dislike_count),
3752                 })
3753         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3754         if vsir:
3755             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3756             info.update({
3757                 'channel': self._get_text(vor, 'title'),
3758                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3759
3760             rows = try_get(
3761                 vsir,
3762                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3763                 list) or []
3764             multiple_songs = False
3765             for row in rows:
3766                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3767                     multiple_songs = True
3768                     break
3769             for row in rows:
3770                 mrr = row.get('metadataRowRenderer') or {}
3771                 mrr_title = mrr.get('title')
3772                 if not mrr_title:
3773                     continue
3774                 mrr_title = self._get_text(mrr, 'title')
3775                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3776                 if mrr_title == 'License':
3777                     info['license'] = mrr_contents_text
3778                 elif not multiple_songs:
3779                     if mrr_title == 'Album':
3780                         info['album'] = mrr_contents_text
3781                     elif mrr_title == 'Artist':
3782                         info['artist'] = mrr_contents_text
3783                     elif mrr_title == 'Song':
3784                         info['track'] = mrr_contents_text
3785
3786         fallbacks = {
3787             'channel': 'uploader',
3788             'channel_id': 'uploader_id',
3789             'channel_url': 'uploader_url',
3790         }
3791
3792         # The upload date for scheduled, live and past live streams / premieres in microformats
3793         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3794         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3795         upload_date = (
3796             unified_strdate(get_first(microformats, 'uploadDate'))
3797             or unified_strdate(search_meta('uploadDate')))
3798         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3799             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3800         info['upload_date'] = upload_date
3801
3802         for to, frm in fallbacks.items():
3803             if not info.get(to):
3804                 info[to] = info.get(frm)
3805
3806         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3807             v = info.get(s_k)
3808             if v:
3809                 info[d_k] = v
3810
3811         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3812         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3813         is_membersonly = None
3814         is_premium = None
3815         if initial_data and is_private is not None:
3816             is_membersonly = False
3817             is_premium = False
3818             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3819             badge_labels = set()
3820             for content in contents:
3821                 if not isinstance(content, dict):
3822                     continue
3823                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3824             for badge_label in badge_labels:
3825                 if badge_label.lower() == 'members only':
3826                     is_membersonly = True
3827                 elif badge_label.lower() == 'premium':
3828                     is_premium = True
3829                 elif badge_label.lower() == 'unlisted':
3830                     is_unlisted = True
3831
3832         info['availability'] = self._availability(
3833             is_private=is_private,
3834             needs_premium=is_premium,
3835             needs_subscription=is_membersonly,
3836             needs_auth=info['age_limit'] >= 18,
3837             is_unlisted=None if is_private is None else is_unlisted)
3838
3839         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3840
3841         self.mark_watched(video_id, player_responses)
3842
3843         return info
3844
3845
3846 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3847
3848     @staticmethod
3849     def passthrough_smuggled_data(func):
3850         def _smuggle(entries, smuggled_data):
3851             for entry in entries:
3852                 # TODO: Convert URL to music.youtube instead.
3853                 # Do we need to passthrough any other smuggled_data?
3854                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3855                 yield entry
3856
3857         @functools.wraps(func)
3858         def wrapper(self, url):
3859             url, smuggled_data = unsmuggle_url(url, {})
3860             if self.is_music_url(url):
3861                 smuggled_data['is_music_url'] = True
3862             info_dict = func(self, url, smuggled_data)
3863             if smuggled_data and info_dict.get('entries'):
3864                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3865             return info_dict
3866         return wrapper
3867
3868     def _extract_channel_id(self, webpage):
3869         channel_id = self._html_search_meta(
3870             'channelId', webpage, 'channel id', default=None)
3871         if channel_id:
3872             return channel_id
3873         channel_url = self._html_search_meta(
3874             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3875              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3876              'twitter:app:url:googleplay'), webpage, 'channel url')
3877         return self._search_regex(
3878             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3879             channel_url, 'channel id')
3880
3881     @staticmethod
3882     def _extract_basic_item_renderer(item):
3883         # Modified from _extract_grid_item_renderer
3884         known_basic_renderers = (
3885             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3886         )
3887         for key, renderer in item.items():
3888             if not isinstance(renderer, dict):
3889                 continue
3890             elif key in known_basic_renderers:
3891                 return renderer
3892             elif key.startswith('grid') and key.endswith('Renderer'):
3893                 return renderer
3894
3895     def _grid_entries(self, grid_renderer):
3896         for item in grid_renderer['items']:
3897             if not isinstance(item, dict):
3898                 continue
3899             renderer = self._extract_basic_item_renderer(item)
3900             if not isinstance(renderer, dict):
3901                 continue
3902             title = self._get_text(renderer, 'title')
3903
3904             # playlist
3905             playlist_id = renderer.get('playlistId')
3906             if playlist_id:
3907                 yield self.url_result(
3908                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3909                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3910                     video_title=title)
3911                 continue
3912             # video
3913             video_id = renderer.get('videoId')
3914             if video_id:
3915                 yield self._extract_video(renderer)
3916                 continue
3917             # channel
3918             channel_id = renderer.get('channelId')
3919             if channel_id:
3920                 yield self.url_result(
3921                     'https://www.youtube.com/channel/%s' % channel_id,
3922                     ie=YoutubeTabIE.ie_key(), video_title=title)
3923                 continue
3924             # generic endpoint URL support
3925             ep_url = urljoin('https://www.youtube.com/', try_get(
3926                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3927                 compat_str))
3928             if ep_url:
3929                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3930                     if ie.suitable(ep_url):
3931                         yield self.url_result(
3932                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3933                         break
3934
3935     def _music_reponsive_list_entry(self, renderer):
3936         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3937         if video_id:
3938             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3939                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3940         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3941         if playlist_id:
3942             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3943             if video_id:
3944                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3945                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3946             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3947                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3948         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3949         if browse_id:
3950             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3951                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3952
3953     def _shelf_entries_from_content(self, shelf_renderer):
3954         content = shelf_renderer.get('content')
3955         if not isinstance(content, dict):
3956             return
3957         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3958         if renderer:
3959             # TODO: add support for nested playlists so each shelf is processed
3960             # as separate playlist
3961             # TODO: this includes only first N items
3962             yield from self._grid_entries(renderer)
3963         renderer = content.get('horizontalListRenderer')
3964         if renderer:
3965             # TODO
3966             pass
3967
3968     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3969         ep = try_get(
3970             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3971             compat_str)
3972         shelf_url = urljoin('https://www.youtube.com', ep)
3973         if shelf_url:
3974             # Skipping links to another channels, note that checking for
3975             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3976             # will not work
3977             if skip_channels and '/channels?' in shelf_url:
3978                 return
3979             title = self._get_text(shelf_renderer, 'title')
3980             yield self.url_result(shelf_url, video_title=title)
3981         # Shelf may not contain shelf URL, fallback to extraction from content
3982         yield from self._shelf_entries_from_content(shelf_renderer)
3983
3984     def _playlist_entries(self, video_list_renderer):
3985         for content in video_list_renderer['contents']:
3986             if not isinstance(content, dict):
3987                 continue
3988             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3989             if not isinstance(renderer, dict):
3990                 continue
3991             video_id = renderer.get('videoId')
3992             if not video_id:
3993                 continue
3994             yield self._extract_video(renderer)
3995
3996     def _rich_entries(self, rich_grid_renderer):
3997         renderer = try_get(
3998             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3999         video_id = renderer.get('videoId')
4000         if not video_id:
4001             return
4002         yield self._extract_video(renderer)
4003
4004     def _video_entry(self, video_renderer):
4005         video_id = video_renderer.get('videoId')
4006         if video_id:
4007             return self._extract_video(video_renderer)
4008
4009     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4010         url = urljoin('https://youtube.com', traverse_obj(
4011             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4012         if url:
4013             return self.url_result(
4014                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4015
4016     def _post_thread_entries(self, post_thread_renderer):
4017         post_renderer = try_get(
4018             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4019         if not post_renderer:
4020             return
4021         # video attachment
4022         video_renderer = try_get(
4023             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4024         video_id = video_renderer.get('videoId')
4025         if video_id:
4026             entry = self._extract_video(video_renderer)
4027             if entry:
4028                 yield entry
4029         # playlist attachment
4030         playlist_id = try_get(
4031             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
4032         if playlist_id:
4033             yield self.url_result(
4034                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4035                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4036         # inline video links
4037         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4038         for run in runs:
4039             if not isinstance(run, dict):
4040                 continue
4041             ep_url = try_get(
4042                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
4043             if not ep_url:
4044                 continue
4045             if not YoutubeIE.suitable(ep_url):
4046                 continue
4047             ep_video_id = YoutubeIE._match_id(ep_url)
4048             if video_id == ep_video_id:
4049                 continue
4050             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4051
4052     def _post_thread_continuation_entries(self, post_thread_continuation):
4053         contents = post_thread_continuation.get('contents')
4054         if not isinstance(contents, list):
4055             return
4056         for content in contents:
4057             renderer = content.get('backstagePostThreadRenderer')
4058             if isinstance(renderer, dict):
4059                 yield from self._post_thread_entries(renderer)
4060                 continue
4061             renderer = content.get('videoRenderer')
4062             if isinstance(renderer, dict):
4063                 yield self._video_entry(renderer)
4064
4065     r''' # unused
4066     def _rich_grid_entries(self, contents):
4067         for content in contents:
4068             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4069             if video_renderer:
4070                 entry = self._video_entry(video_renderer)
4071                 if entry:
4072                     yield entry
4073     '''
4074
4075     def _extract_entries(self, parent_renderer, continuation_list):
4076         # continuation_list is modified in-place with continuation_list = [continuation_token]
4077         continuation_list[:] = [None]
4078         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4079         for content in contents:
4080             if not isinstance(content, dict):
4081                 continue
4082             is_renderer = traverse_obj(
4083                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4084                 expected_type=dict)
4085             if not is_renderer:
4086                 renderer = content.get('richItemRenderer')
4087                 if renderer:
4088                     for entry in self._rich_entries(renderer):
4089                         yield entry
4090                     continuation_list[0] = self._extract_continuation(parent_renderer)
4091                 continue
4092             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4093             for isr_content in isr_contents:
4094                 if not isinstance(isr_content, dict):
4095                     continue
4096
4097                 known_renderers = {
4098                     'playlistVideoListRenderer': self._playlist_entries,
4099                     'gridRenderer': self._grid_entries,
4100                     'reelShelfRenderer': self._grid_entries,
4101                     'shelfRenderer': self._shelf_entries,
4102                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4103                     'backstagePostThreadRenderer': self._post_thread_entries,
4104                     'videoRenderer': lambda x: [self._video_entry(x)],
4105                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4106                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4107                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4108                 }
4109                 for key, renderer in isr_content.items():
4110                     if key not in known_renderers:
4111                         continue
4112                     for entry in known_renderers[key](renderer):
4113                         if entry:
4114                             yield entry
4115                     continuation_list[0] = self._extract_continuation(renderer)
4116                     break
4117
4118             if not continuation_list[0]:
4119                 continuation_list[0] = self._extract_continuation(is_renderer)
4120
4121         if not continuation_list[0]:
4122             continuation_list[0] = self._extract_continuation(parent_renderer)
4123
4124     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4125         continuation_list = [None]
4126         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4127         tab_content = try_get(tab, lambda x: x['content'], dict)
4128         if not tab_content:
4129             return
4130         parent_renderer = (
4131             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4132             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4133         yield from extract_entries(parent_renderer)
4134         continuation = continuation_list[0]
4135
4136         for page_num in itertools.count(1):
4137             if not continuation:
4138                 break
4139             headers = self.generate_api_headers(
4140                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4141             response = self._extract_response(
4142                 item_id=f'{item_id} page {page_num}',
4143                 query=continuation, headers=headers, ytcfg=ytcfg,
4144                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4145
4146             if not response:
4147                 break
4148             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4149             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4150             visitor_data = self._extract_visitor_data(response) or visitor_data
4151
4152             known_continuation_renderers = {
4153                 'playlistVideoListContinuation': self._playlist_entries,
4154                 'gridContinuation': self._grid_entries,
4155                 'itemSectionContinuation': self._post_thread_continuation_entries,
4156                 'sectionListContinuation': extract_entries,  # for feeds
4157             }
4158             continuation_contents = try_get(
4159                 response, lambda x: x['continuationContents'], dict) or {}
4160             continuation_renderer = None
4161             for key, value in continuation_contents.items():
4162                 if key not in known_continuation_renderers:
4163                     continue
4164                 continuation_renderer = value
4165                 continuation_list = [None]
4166                 yield from known_continuation_renderers[key](continuation_renderer)
4167                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4168                 break
4169             if continuation_renderer:
4170                 continue
4171
4172             known_renderers = {
4173                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4174                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4175                 'gridVideoRenderer': (self._grid_entries, 'items'),
4176                 'gridChannelRenderer': (self._grid_entries, 'items'),
4177                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4178                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4179                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4180                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4181             }
4182             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4183             continuation_items = try_get(
4184                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4185             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4186             video_items_renderer = None
4187             for key, value in continuation_item.items():
4188                 if key not in known_renderers:
4189                     continue
4190                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4191                 continuation_list = [None]
4192                 yield from known_renderers[key][0](video_items_renderer)
4193                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4194                 break
4195             if video_items_renderer:
4196                 continue
4197             break
4198
4199     @staticmethod
4200     def _extract_selected_tab(tabs, fatal=True):
4201         for tab in tabs:
4202             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4203             if renderer.get('selected') is True:
4204                 return renderer
4205         else:
4206             if fatal:
4207                 raise ExtractorError('Unable to find selected tab')
4208
4209     def _extract_uploader(self, data):
4210         uploader = {}
4211         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4212         owner = try_get(
4213             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4214         if owner:
4215             owner_text = owner.get('text')
4216             uploader['uploader'] = self._search_regex(
4217                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4218             uploader['uploader_id'] = try_get(
4219                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
4220             uploader['uploader_url'] = urljoin(
4221                 'https://www.youtube.com/',
4222                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
4223         return {k: v for k, v in uploader.items() if v is not None}
4224
4225     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4226         playlist_id = title = description = channel_url = channel_name = channel_id = None
4227         tags = []
4228
4229         selected_tab = self._extract_selected_tab(tabs)
4230         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4231         renderer = try_get(
4232             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4233         if renderer:
4234             channel_name = renderer.get('title')
4235             channel_url = renderer.get('channelUrl')
4236             channel_id = renderer.get('externalId')
4237         else:
4238             renderer = try_get(
4239                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4240
4241         if renderer:
4242             title = renderer.get('title')
4243             description = renderer.get('description', '')
4244             playlist_id = channel_id
4245             tags = renderer.get('keywords', '').split()
4246
4247         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4248         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4249         def _get_uncropped(url):
4250             return url_or_none((url or '').split('=')[0] + '=s0')
4251
4252         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4253         if avatar_thumbnails:
4254             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4255             if uncropped_avatar:
4256                 avatar_thumbnails.append({
4257                     'url': uncropped_avatar,
4258                     'id': 'avatar_uncropped',
4259                     'preference': 1
4260                 })
4261
4262         channel_banners = self._extract_thumbnails(
4263             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4264         for banner in channel_banners:
4265             banner['preference'] = -10
4266
4267         if channel_banners:
4268             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4269             if uncropped_banner:
4270                 channel_banners.append({
4271                     'url': uncropped_banner,
4272                     'id': 'banner_uncropped',
4273                     'preference': -5
4274                 })
4275
4276         primary_thumbnails = self._extract_thumbnails(
4277             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4278
4279         if playlist_id is None:
4280             playlist_id = item_id
4281
4282         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4283         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4284         if title is None:
4285             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4286         title += format_field(selected_tab, 'title', ' - %s')
4287         title += format_field(selected_tab, 'expandedText', ' - %s')
4288
4289         metadata = {
4290             'playlist_id': playlist_id,
4291             'playlist_title': title,
4292             'playlist_description': description,
4293             'uploader': channel_name,
4294             'uploader_id': channel_id,
4295             'uploader_url': channel_url,
4296             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4297             'tags': tags,
4298             'view_count': self._get_count(playlist_stats, 1),
4299             'availability': self._extract_availability(data),
4300             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4301             'playlist_count': self._get_count(playlist_stats, 0),
4302             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4303         }
4304         if not channel_id:
4305             metadata.update(self._extract_uploader(data))
4306         metadata.update({
4307             'channel': metadata['uploader'],
4308             'channel_id': metadata['uploader_id'],
4309             'channel_url': metadata['uploader_url']})
4310         return self.playlist_result(
4311             self._entries(
4312                 selected_tab, playlist_id, ytcfg,
4313                 self._extract_account_syncid(ytcfg, data),
4314                 self._extract_visitor_data(data, ytcfg)),
4315             **metadata)
4316
4317     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4318         first_id = last_id = response = None
4319         for page_num in itertools.count(1):
4320             videos = list(self._playlist_entries(playlist))
4321             if not videos:
4322                 return
4323             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4324             if start >= len(videos):
4325                 return
4326             yield from videos[start:]
4327             first_id = first_id or videos[0]['id']
4328             last_id = videos[-1]['id']
4329             watch_endpoint = try_get(
4330                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4331             headers = self.generate_api_headers(
4332                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4333                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4334             query = {
4335                 'playlistId': playlist_id,
4336                 'videoId': watch_endpoint.get('videoId') or last_id,
4337                 'index': watch_endpoint.get('index') or len(videos),
4338                 'params': watch_endpoint.get('params') or 'OAE%3D'
4339             }
4340             response = self._extract_response(
4341                 item_id='%s page %d' % (playlist_id, page_num),
4342                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4343                 check_get_keys='contents'
4344             )
4345             playlist = try_get(
4346                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4347
4348     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4349         title = playlist.get('title') or try_get(
4350             data, lambda x: x['titleText']['simpleText'], compat_str)
4351         playlist_id = playlist.get('playlistId') or item_id
4352
4353         # Delegating everything except mix playlists to regular tab-based playlist URL
4354         playlist_url = urljoin(url, try_get(
4355             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4356             compat_str))
4357
4358         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4359         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4360         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4361
4362         if playlist_url and playlist_url != url and not is_known_unviewable:
4363             return self.url_result(
4364                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4365                 video_title=title)
4366
4367         return self.playlist_result(
4368             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4369             playlist_id=playlist_id, playlist_title=title)
4370
4371     def _extract_availability(self, data):
4372         """
4373         Gets the availability of a given playlist/tab.
4374         Note: Unless YouTube tells us explicitly, we do not assume it is public
4375         @param data: response
4376         """
4377         is_private = is_unlisted = None
4378         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4379         badge_labels = self._extract_badges(renderer)
4380
4381         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4382         privacy_dropdown_entries = try_get(
4383             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4384         for renderer_dict in privacy_dropdown_entries:
4385             is_selected = try_get(
4386                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4387             if not is_selected:
4388                 continue
4389             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4390             if label:
4391                 badge_labels.add(label.lower())
4392                 break
4393
4394         for badge_label in badge_labels:
4395             if badge_label == 'unlisted':
4396                 is_unlisted = True
4397             elif badge_label == 'private':
4398                 is_private = True
4399             elif badge_label == 'public':
4400                 is_unlisted = is_private = False
4401         return self._availability(is_private, False, False, False, is_unlisted)
4402
4403     @staticmethod
4404     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4405         sidebar_renderer = try_get(
4406             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4407         for item in sidebar_renderer:
4408             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4409             if renderer:
4410                 return renderer
4411
4412     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4413         """
4414         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4415         """
4416         browse_id = params = None
4417         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4418         if not renderer:
4419             return
4420         menu_renderer = try_get(
4421             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4422         for menu_item in menu_renderer:
4423             if not isinstance(menu_item, dict):
4424                 continue
4425             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4426             text = try_get(
4427                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4428             if not text or text.lower() != 'show unavailable videos':
4429                 continue
4430             browse_endpoint = try_get(
4431                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4432             browse_id = browse_endpoint.get('browseId')
4433             params = browse_endpoint.get('params')
4434             break
4435
4436         headers = self.generate_api_headers(
4437             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4438             visitor_data=self._extract_visitor_data(data, ytcfg))
4439         query = {
4440             'params': params or 'wgYCCAA=',
4441             'browseId': browse_id or 'VL%s' % item_id
4442         }
4443         return self._extract_response(
4444             item_id=item_id, headers=headers, query=query,
4445             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4446             note='Downloading API JSON with unavailable videos')
4447
4448     @functools.cached_property
4449     def skip_webpage(self):
4450         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4451
4452     def _extract_webpage(self, url, item_id, fatal=True):
4453         retries = self.get_param('extractor_retries', 3)
4454         count = -1
4455         webpage = data = last_error = None
4456         while count < retries:
4457             count += 1
4458             # Sometimes youtube returns a webpage with incomplete ytInitialData
4459             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4460             if last_error:
4461                 self.report_warning('%s. Retrying ...' % last_error)
4462             try:
4463                 webpage = self._download_webpage(
4464                     url, item_id,
4465                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4466                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4467             except ExtractorError as e:
4468                 if isinstance(e.cause, network_exceptions):
4469                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4470                         last_error = error_to_compat_str(e.cause or e.msg)
4471                         if count < retries:
4472                             continue
4473                 if fatal:
4474                     raise
4475                 self.report_warning(error_to_compat_str(e))
4476                 break
4477             else:
4478                 try:
4479                     self._extract_and_report_alerts(data)
4480                 except ExtractorError as e:
4481                     if fatal:
4482                         raise
4483                     self.report_warning(error_to_compat_str(e))
4484                     break
4485
4486                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4487                     break
4488
4489                 last_error = 'Incomplete yt initial data received'
4490                 if count >= retries:
4491                     if fatal:
4492                         raise ExtractorError(last_error)
4493                     self.report_warning(last_error)
4494                     break
4495
4496         return webpage, data
4497
4498     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4499         """Use if failed to extract ytcfg (and data) from initial webpage"""
4500         if not ytcfg and self.is_authenticated:
4501             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4502             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4503                 raise ExtractorError(
4504                     f'{msg}. If you are not downloading private content, or '
4505                     'your cookies are only for the first account and channel,'
4506                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4507                     expected=True)
4508             self.report_warning(msg, only_once=True)
4509
4510     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4511         data = None
4512         if not self.skip_webpage:
4513             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4514             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4515             # Reject webpage data if redirected to home page without explicitly requesting
4516             selected_tab = self._extract_selected_tab(traverse_obj(
4517                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4518             if (url != 'https://www.youtube.com/feed/recommended'
4519                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4520                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4521                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4522                 if fatal:
4523                     raise ExtractorError(msg, expected=True)
4524                 self.report_warning(msg, only_once=True)
4525         if not data:
4526             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4527             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4528         return data, ytcfg
4529
4530     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4531         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4532         resolve_response = self._extract_response(
4533             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4534             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4535         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4536         for ep_key, ep in endpoints.items():
4537             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4538             if params:
4539                 return self._extract_response(
4540                     item_id=item_id, query=params, ep=ep, headers=headers,
4541                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4542                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4543         err_note = 'Failed to resolve url (does the playlist exist?)'
4544         if fatal:
4545             raise ExtractorError(err_note, expected=True)
4546         self.report_warning(err_note, item_id)
4547
4548     _SEARCH_PARAMS = None
4549
4550     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4551         data = {'query': query}
4552         if params is NO_DEFAULT:
4553             params = self._SEARCH_PARAMS
4554         if params:
4555             data['params'] = params
4556
4557         content_keys = (
4558             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4559             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4560             # ytmusic search
4561             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4562             ('continuationContents', ),
4563         )
4564         display_id = f'query "{query}"'
4565         check_get_keys = tuple({keys[0] for keys in content_keys})
4566         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4567         self._report_playlist_authcheck(ytcfg, fatal=False)
4568
4569         continuation_list = [None]
4570         search = None
4571         for page_num in itertools.count(1):
4572             data.update(continuation_list[0] or {})
4573             headers = self.generate_api_headers(
4574                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4575             search = self._extract_response(
4576                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4577                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4578             slr_contents = traverse_obj(search, *content_keys)
4579             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4580             if not continuation_list[0]:
4581                 break
4582
4583
4584 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4585     IE_DESC = 'YouTube Tabs'
4586     _VALID_URL = r'''(?x:
4587         https?://
4588             (?:\w+\.)?
4589             (?:
4590                 youtube(?:kids)?\.com|
4591                 %(invidious)s
4592             )/
4593             (?:
4594                 (?P<channel_type>channel|c|user|browse)/|
4595                 (?P<not_channel>
4596                     feed/|hashtag/|
4597                     (?:playlist|watch)\?.*?\blist=
4598                 )|
4599                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4600             )
4601             (?P<id>[^/?\#&]+)
4602     )''' % {
4603         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4604         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4605     }
4606     IE_NAME = 'youtube:tab'
4607
4608     _TESTS = [{
4609         'note': 'playlists, multipage',
4610         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4611         'playlist_mincount': 94,
4612         'info_dict': {
4613             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4614             'title': 'Igor Kleiner - Playlists',
4615             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4616             'uploader': 'Igor Kleiner',
4617             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4618             'channel': 'Igor Kleiner',
4619             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4620             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4621             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4622             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4623             'channel_follower_count': int
4624         },
4625     }, {
4626         'note': 'playlists, multipage, different order',
4627         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4628         'playlist_mincount': 94,
4629         'info_dict': {
4630             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4631             'title': 'Igor Kleiner - Playlists',
4632             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4633             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4634             'uploader': 'Igor Kleiner',
4635             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4636             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4637             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4638             'channel': 'Igor Kleiner',
4639             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4640             'channel_follower_count': int
4641         },
4642     }, {
4643         'note': 'playlists, series',
4644         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4645         'playlist_mincount': 5,
4646         'info_dict': {
4647             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4648             'title': '3Blue1Brown - Playlists',
4649             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4650             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4651             'uploader': '3Blue1Brown',
4652             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4653             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4654             'channel': '3Blue1Brown',
4655             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4656             'tags': ['Mathematics'],
4657             'channel_follower_count': int
4658         },
4659     }, {
4660         'note': 'playlists, singlepage',
4661         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4662         'playlist_mincount': 4,
4663         'info_dict': {
4664             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4665             'title': 'ThirstForScience - Playlists',
4666             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4667             'uploader': 'ThirstForScience',
4668             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4669             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4670             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4671             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4672             'tags': 'count:13',
4673             'channel': 'ThirstForScience',
4674             'channel_follower_count': int
4675         }
4676     }, {
4677         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4678         'only_matching': True,
4679     }, {
4680         'note': 'basic, single video playlist',
4681         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4682         'info_dict': {
4683             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4684             'uploader': 'Sergey M.',
4685             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4686             'title': 'youtube-dl public playlist',
4687             'description': '',
4688             'tags': [],
4689             'view_count': int,
4690             'modified_date': '20201130',
4691             'channel': 'Sergey M.',
4692             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4693             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4694             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4695         },
4696         'playlist_count': 1,
4697     }, {
4698         'note': 'empty playlist',
4699         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4700         'info_dict': {
4701             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4702             'uploader': 'Sergey M.',
4703             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4704             'title': 'youtube-dl empty playlist',
4705             'tags': [],
4706             'channel': 'Sergey M.',
4707             'description': '',
4708             'modified_date': '20160902',
4709             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4710             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4711             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4712         },
4713         'playlist_count': 0,
4714     }, {
4715         'note': 'Home tab',
4716         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4717         'info_dict': {
4718             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4719             'title': 'lex will - Home',
4720             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4721             'uploader': 'lex will',
4722             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4723             'channel': 'lex will',
4724             'tags': ['bible', 'history', 'prophesy'],
4725             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4726             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4727             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4728             'channel_follower_count': int
4729         },
4730         'playlist_mincount': 2,
4731     }, {
4732         'note': 'Videos tab',
4733         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4734         'info_dict': {
4735             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4736             'title': 'lex will - Videos',
4737             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4738             'uploader': 'lex will',
4739             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4740             'tags': ['bible', 'history', 'prophesy'],
4741             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4742             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4743             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4744             'channel': 'lex will',
4745             'channel_follower_count': int
4746         },
4747         'playlist_mincount': 975,
4748     }, {
4749         'note': 'Videos tab, sorted by popular',
4750         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4751         'info_dict': {
4752             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4753             'title': 'lex will - Videos',
4754             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4755             'uploader': 'lex will',
4756             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4757             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4758             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4759             'channel': 'lex will',
4760             'tags': ['bible', 'history', 'prophesy'],
4761             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4762             'channel_follower_count': int
4763         },
4764         'playlist_mincount': 199,
4765     }, {
4766         'note': 'Playlists tab',
4767         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4768         'info_dict': {
4769             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4770             'title': 'lex will - Playlists',
4771             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4772             'uploader': 'lex will',
4773             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4774             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4775             'channel': 'lex will',
4776             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4777             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4778             'tags': ['bible', 'history', 'prophesy'],
4779             'channel_follower_count': int
4780         },
4781         'playlist_mincount': 17,
4782     }, {
4783         'note': 'Community tab',
4784         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4785         'info_dict': {
4786             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4787             'title': 'lex will - Community',
4788             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4789             'uploader': 'lex will',
4790             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4791             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4792             'channel': 'lex will',
4793             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4794             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4795             'tags': ['bible', 'history', 'prophesy'],
4796             'channel_follower_count': int
4797         },
4798         'playlist_mincount': 18,
4799     }, {
4800         'note': 'Channels tab',
4801         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4802         'info_dict': {
4803             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4804             'title': 'lex will - Channels',
4805             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4806             'uploader': 'lex will',
4807             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4808             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4809             'channel': 'lex will',
4810             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4811             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4812             'tags': ['bible', 'history', 'prophesy'],
4813             'channel_follower_count': int
4814         },
4815         'playlist_mincount': 12,
4816     }, {
4817         'note': 'Search tab',
4818         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4819         'playlist_mincount': 40,
4820         'info_dict': {
4821             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4822             'title': '3Blue1Brown - Search - linear algebra',
4823             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4824             'uploader': '3Blue1Brown',
4825             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4826             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4827             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4828             'tags': ['Mathematics'],
4829             'channel': '3Blue1Brown',
4830             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4831             'channel_follower_count': int
4832         },
4833     }, {
4834         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4835         'only_matching': True,
4836     }, {
4837         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4838         'only_matching': True,
4839     }, {
4840         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4841         'only_matching': True,
4842     }, {
4843         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4844         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4845         'info_dict': {
4846             'title': '29C3: Not my department',
4847             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4848             'uploader': 'Christiaan008',
4849             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4850             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4851             'tags': [],
4852             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4853             'view_count': int,
4854             'modified_date': '20150605',
4855             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4856             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4857             'channel': 'Christiaan008',
4858         },
4859         'playlist_count': 96,
4860     }, {
4861         'note': 'Large playlist',
4862         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4863         'info_dict': {
4864             'title': 'Uploads from Cauchemar',
4865             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4866             'uploader': 'Cauchemar',
4867             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4868             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4869             'tags': [],
4870             'modified_date': r're:\d{8}',
4871             'channel': 'Cauchemar',
4872             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4873             'view_count': int,
4874             'description': '',
4875             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4876         },
4877         'playlist_mincount': 1123,
4878         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4879     }, {
4880         'note': 'even larger playlist, 8832 videos',
4881         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4882         'only_matching': True,
4883     }, {
4884         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4885         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4886         'info_dict': {
4887             'title': 'Uploads from Interstellar Movie',
4888             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4889             'uploader': 'Interstellar Movie',
4890             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4891             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4892             'tags': [],
4893             'view_count': int,
4894             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4895             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4896             'channel': 'Interstellar Movie',
4897             'description': '',
4898             'modified_date': r're:\d{8}',
4899         },
4900         'playlist_mincount': 21,
4901     }, {
4902         'note': 'Playlist with "show unavailable videos" button',
4903         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4904         'info_dict': {
4905             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4906             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4907             'uploader': 'Phim Siêu Nhân Nhật Bản',
4908             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4909             'view_count': int,
4910             'channel': 'Phim Siêu Nhân Nhật Bản',
4911             'tags': [],
4912             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4913             'description': '',
4914             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4915             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4916             'modified_date': r're:\d{8}',
4917         },
4918         'playlist_mincount': 200,
4919         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4920     }, {
4921         'note': 'Playlist with unavailable videos in page 7',
4922         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4923         'info_dict': {
4924             'title': 'Uploads from BlankTV',
4925             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4926             'uploader': 'BlankTV',
4927             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4928             'channel': 'BlankTV',
4929             'channel_url': 'https://www.youtube.com/c/blanktv',
4930             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4931             'view_count': int,
4932             'tags': [],
4933             'uploader_url': 'https://www.youtube.com/c/blanktv',
4934             'modified_date': r're:\d{8}',
4935             'description': '',
4936         },
4937         'playlist_mincount': 1000,
4938         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4939     }, {
4940         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4941         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4942         'info_dict': {
4943             'title': 'Data Analysis with Dr Mike Pound',
4944             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4945             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4946             'uploader': 'Computerphile',
4947             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4948             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4949             'tags': [],
4950             'view_count': int,
4951             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4952             'channel_url': 'https://www.youtube.com/user/Computerphile',
4953             'channel': 'Computerphile',
4954         },
4955         'playlist_mincount': 11,
4956     }, {
4957         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4958         'only_matching': True,
4959     }, {
4960         'note': 'Playlist URL that does not actually serve a playlist',
4961         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4962         'info_dict': {
4963             'id': 'FqZTN594JQw',
4964             'ext': 'webm',
4965             'title': "Smiley's People 01 detective, Adventure Series, Action",
4966             'uploader': 'STREEM',
4967             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4968             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4969             'upload_date': '20150526',
4970             'license': 'Standard YouTube License',
4971             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4972             'categories': ['People & Blogs'],
4973             'tags': list,
4974             'view_count': int,
4975             'like_count': int,
4976         },
4977         'params': {
4978             'skip_download': True,
4979         },
4980         'skip': 'This video is not available.',
4981         'add_ie': [YoutubeIE.ie_key()],
4982     }, {
4983         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
4984         'only_matching': True,
4985     }, {
4986         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
4987         'only_matching': True,
4988     }, {
4989         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
4990         'info_dict': {
4991             'id': 'GgL890LIznQ',  # This will keep changing
4992             'ext': 'mp4',
4993             'title': str,
4994             'uploader': 'Sky News',
4995             'uploader_id': 'skynews',
4996             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
4997             'upload_date': r're:\d{8}',
4998             'description': str,
4999             'categories': ['News & Politics'],
5000             'tags': list,
5001             'like_count': int,
5002             'release_timestamp': 1642502819,
5003             'channel': 'Sky News',
5004             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5005             'age_limit': 0,
5006             'view_count': int,
5007             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5008             'playable_in_embed': True,
5009             'release_date': '20220118',
5010             'availability': 'public',
5011             'live_status': 'is_live',
5012             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5013             'channel_follower_count': int
5014         },
5015         'params': {
5016             'skip_download': True,
5017         },
5018         'expected_warnings': ['Ignoring subtitle tracks found in '],
5019     }, {
5020         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5021         'info_dict': {
5022             'id': 'a48o2S1cPoo',
5023             'ext': 'mp4',
5024             'title': 'The Young Turks - Live Main Show',
5025             'uploader': 'The Young Turks',
5026             'uploader_id': 'TheYoungTurks',
5027             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5028             'upload_date': '20150715',
5029             'license': 'Standard YouTube License',
5030             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5031             'categories': ['News & Politics'],
5032             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5033             'like_count': int,
5034         },
5035         'params': {
5036             'skip_download': True,
5037         },
5038         'only_matching': True,
5039     }, {
5040         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5041         'only_matching': True,
5042     }, {
5043         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5044         'only_matching': True,
5045     }, {
5046         'note': 'A channel that is not live. Should raise error',
5047         'url': 'https://www.youtube.com/user/numberphile/live',
5048         'only_matching': True,
5049     }, {
5050         'url': 'https://www.youtube.com/feed/trending',
5051         'only_matching': True,
5052     }, {
5053         'url': 'https://www.youtube.com/feed/library',
5054         'only_matching': True,
5055     }, {
5056         'url': 'https://www.youtube.com/feed/history',
5057         'only_matching': True,
5058     }, {
5059         'url': 'https://www.youtube.com/feed/subscriptions',
5060         'only_matching': True,
5061     }, {
5062         'url': 'https://www.youtube.com/feed/watch_later',
5063         'only_matching': True,
5064     }, {
5065         'note': 'Recommended - redirects to home page.',
5066         'url': 'https://www.youtube.com/feed/recommended',
5067         'only_matching': True,
5068     }, {
5069         'note': 'inline playlist with not always working continuations',
5070         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5071         'only_matching': True,
5072     }, {
5073         'url': 'https://www.youtube.com/course',
5074         'only_matching': True,
5075     }, {
5076         'url': 'https://www.youtube.com/zsecurity',
5077         'only_matching': True,
5078     }, {
5079         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5080         'only_matching': True,
5081     }, {
5082         'url': 'https://www.youtube.com/TheYoungTurks/live',
5083         'only_matching': True,
5084     }, {
5085         'url': 'https://www.youtube.com/hashtag/cctv9',
5086         'info_dict': {
5087             'id': 'cctv9',
5088             'title': '#cctv9',
5089             'tags': [],
5090         },
5091         'playlist_mincount': 350,
5092     }, {
5093         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5094         'only_matching': True,
5095     }, {
5096         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5097         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5098         'only_matching': True
5099     }, {
5100         'note': '/browse/ should redirect to /channel/',
5101         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5102         'only_matching': True
5103     }, {
5104         'note': 'VLPL, should redirect to playlist?list=PL...',
5105         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5106         'info_dict': {
5107             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5108             'uploader': 'NoCopyrightSounds',
5109             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5110             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5111             'title': 'NCS Releases',
5112             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5113             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5114             'modified_date': r're:\d{8}',
5115             'view_count': int,
5116             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5117             'tags': [],
5118             'channel': 'NoCopyrightSounds',
5119         },
5120         'playlist_mincount': 166,
5121         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5122     }, {
5123         'note': 'Topic, should redirect to playlist?list=UU...',
5124         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5125         'info_dict': {
5126             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5127             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5128             'title': 'Uploads from Royalty Free Music - Topic',
5129             'uploader': 'Royalty Free Music - Topic',
5130             'tags': [],
5131             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5132             'channel': 'Royalty Free Music - Topic',
5133             'view_count': int,
5134             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5135             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5136             'modified_date': r're:\d{8}',
5137             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5138             'description': '',
5139         },
5140         'expected_warnings': [
5141             'The URL does not have a videos tab',
5142             r'[Uu]navailable videos (are|will be) hidden',
5143         ],
5144         'playlist_mincount': 101,
5145     }, {
5146         'note': 'Topic without a UU playlist',
5147         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5148         'info_dict': {
5149             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5150             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5151             'tags': [],
5152         },
5153         'expected_warnings': [
5154             'the playlist redirect gave error',
5155         ],
5156         'playlist_mincount': 9,
5157     }, {
5158         'note': 'Youtube music Album',
5159         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5160         'info_dict': {
5161             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5162             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5163             'tags': [],
5164             'view_count': int,
5165             'description': '',
5166             'availability': 'unlisted',
5167             'modified_date': r're:\d{8}',
5168         },
5169         'playlist_count': 50,
5170     }, {
5171         'note': 'unlisted single video playlist',
5172         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5173         'info_dict': {
5174             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5175             'uploader': 'colethedj',
5176             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5177             'title': 'yt-dlp unlisted playlist test',
5178             'availability': 'unlisted',
5179             'tags': [],
5180             'modified_date': '20211208',
5181             'channel': 'colethedj',
5182             'view_count': int,
5183             'description': '',
5184             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5185             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5186             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5187         },
5188         'playlist_count': 1,
5189     }, {
5190         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5191         'url': 'https://www.youtube.com/feed/recommended',
5192         'info_dict': {
5193             'id': 'recommended',
5194             'title': 'recommended',
5195             'tags': [],
5196         },
5197         'playlist_mincount': 50,
5198         'params': {
5199             'skip_download': True,
5200             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5201         },
5202     }, {
5203         'note': 'API Fallback: /videos tab, sorted by oldest first',
5204         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5205         'info_dict': {
5206             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5207             'title': 'Cody\'sLab - Videos',
5208             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5209             'uploader': 'Cody\'sLab',
5210             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5211             'channel': 'Cody\'sLab',
5212             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5213             'tags': [],
5214             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5215             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5216             'channel_follower_count': int
5217         },
5218         'playlist_mincount': 650,
5219         'params': {
5220             'skip_download': True,
5221             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5222         },
5223     }, {
5224         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5225         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5226         'info_dict': {
5227             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5228             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5229             'title': 'Uploads from Royalty Free Music - Topic',
5230             'uploader': 'Royalty Free Music - Topic',
5231             'modified_date': r're:\d{8}',
5232             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5233             'description': '',
5234             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5235             'tags': [],
5236             'channel': 'Royalty Free Music - Topic',
5237             'view_count': int,
5238             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5239         },
5240         'expected_warnings': [
5241             'does not have a videos tab',
5242             r'[Uu]navailable videos (are|will be) hidden',
5243         ],
5244         'playlist_mincount': 101,
5245         'params': {
5246             'skip_download': True,
5247             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5248         },
5249     }, {
5250         'note': 'non-standard redirect to regional channel',
5251         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5252         'only_matching': True
5253     }, {
5254         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5255         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5256         'info_dict': {
5257             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5258             'modified_date': '20220407',
5259             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5260             'tags': [],
5261             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5262             'uploader': 'pukkandan',
5263             'availability': 'unlisted',
5264             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5265             'channel': 'pukkandan',
5266             'description': 'Test for collaborative playlist',
5267             'title': 'yt-dlp test - collaborative playlist',
5268             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5269         },
5270         'playlist_mincount': 2
5271     }]
5272
5273     @classmethod
5274     def suitable(cls, url):
5275         return False if YoutubeIE.suitable(url) else super().suitable(url)
5276
5277     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5278
5279     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5280     def _real_extract(self, url, smuggled_data):
5281         item_id = self._match_id(url)
5282         url = compat_urlparse.urlunparse(
5283             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
5284         compat_opts = self.get_param('compat_opts', [])
5285
5286         def get_mobj(url):
5287             mobj = self._URL_RE.match(url).groupdict()
5288             mobj.update((k, '') for k, v in mobj.items() if v is None)
5289             return mobj
5290
5291         mobj, redirect_warning = get_mobj(url), None
5292         # Youtube returns incomplete data if tabname is not lower case
5293         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5294         if is_channel:
5295             if smuggled_data.get('is_music_url'):
5296                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5297                     item_id = item_id[2:]
5298                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5299                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5300                     mdata = self._extract_tab_endpoint(
5301                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5302                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5303                                         get_all=False, expected_type=compat_str)
5304                     if not murl:
5305                         raise ExtractorError('Failed to resolve album to playlist')
5306                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5307                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5308                     pre = f'https://www.youtube.com/channel/{item_id}'
5309
5310         original_tab_name = tab
5311         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5312             # Home URLs should redirect to /videos/
5313             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5314                                 'To download only the videos in the home page, add a "/featured" to the URL')
5315             tab = '/videos'
5316
5317         url = ''.join((pre, tab, post))
5318         mobj = get_mobj(url)
5319
5320         # Handle both video/playlist URLs
5321         qs = parse_qs(url)
5322         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5323
5324         if not video_id and mobj['not_channel'].startswith('watch'):
5325             if not playlist_id:
5326                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5327                 raise ExtractorError('Unable to recognize tab page')
5328             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5329             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5330             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5331             mobj = get_mobj(url)
5332
5333         if video_id and playlist_id:
5334             if self.get_param('noplaylist'):
5335                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5336                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5337                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5338             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5339
5340         data, ytcfg = self._extract_data(url, item_id)
5341
5342         # YouTube may provide a non-standard redirect to the regional channel
5343         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5344         redirect_url = traverse_obj(
5345             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5346         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5347             redirect_url = ''.join((
5348                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5349             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5350             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5351
5352         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5353         if tabs:
5354             selected_tab = self._extract_selected_tab(tabs)
5355             selected_tab_name = selected_tab.get('title', '').lower()
5356             if selected_tab_name == 'home':
5357                 selected_tab_name = 'featured'
5358             requested_tab_name = mobj['tab'][1:]
5359             if 'no-youtube-channel-redirect' not in compat_opts:
5360                 if requested_tab_name == 'live':
5361                     # Live tab should have redirected to the video
5362                     raise ExtractorError('The channel is not currently live', expected=True)
5363                 if requested_tab_name not in ('', selected_tab_name):
5364                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5365                     if not original_tab_name:
5366                         if item_id[:2] == 'UC':
5367                             # Topic channels don't have /videos. Use the equivalent playlist instead
5368                             pl_id = f'UU{item_id[2:]}'
5369                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5370                             try:
5371                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5372                             except ExtractorError:
5373                                 redirect_warning += ' and the playlist redirect gave error'
5374                             else:
5375                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5376                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5377                         if selected_tab_name and selected_tab_name != requested_tab_name:
5378                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5379                     else:
5380                         raise ExtractorError(redirect_warning, expected=True)
5381
5382         if redirect_warning:
5383             self.to_screen(redirect_warning)
5384         self.write_debug(f'Final URL: {url}')
5385
5386         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5387         if 'no-youtube-unavailable-videos' not in compat_opts:
5388             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5389         self._extract_and_report_alerts(data, only_once=True)
5390         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5391         if tabs:
5392             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5393
5394         playlist = traverse_obj(
5395             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5396         if playlist:
5397             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5398
5399         video_id = traverse_obj(
5400             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5401         if video_id:
5402             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5403                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5404             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5405                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5406
5407         raise ExtractorError('Unable to recognize tab page')
5408
5409
5410 class YoutubePlaylistIE(InfoExtractor):
5411     IE_DESC = 'YouTube playlists'
5412     _VALID_URL = r'''(?x)(?:
5413                         (?:https?://)?
5414                         (?:\w+\.)?
5415                         (?:
5416                             (?:
5417                                 youtube(?:kids)?\.com|
5418                                 %(invidious)s
5419                             )
5420                             /.*?\?.*?\blist=
5421                         )?
5422                         (?P<id>%(playlist_id)s)
5423                      )''' % {
5424         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5425         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5426     }
5427     IE_NAME = 'youtube:playlist'
5428     _TESTS = [{
5429         'note': 'issue #673',
5430         'url': 'PLBB231211A4F62143',
5431         'info_dict': {
5432             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5433             'id': 'PLBB231211A4F62143',
5434             'uploader': 'Wickman',
5435             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5436             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5437             'view_count': int,
5438             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5439             'modified_date': r're:\d{8}',
5440             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5441             'channel': 'Wickman',
5442             'tags': [],
5443             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5444         },
5445         'playlist_mincount': 29,
5446     }, {
5447         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5448         'info_dict': {
5449             'title': 'YDL_safe_search',
5450             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5451         },
5452         'playlist_count': 2,
5453         'skip': 'This playlist is private',
5454     }, {
5455         'note': 'embedded',
5456         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5457         'playlist_count': 4,
5458         'info_dict': {
5459             'title': 'JODA15',
5460             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5461             'uploader': 'milan',
5462             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5463             'description': '',
5464             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5465             'tags': [],
5466             'modified_date': '20140919',
5467             'view_count': int,
5468             'channel': 'milan',
5469             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5470             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5471         },
5472         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5473     }, {
5474         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5475         'playlist_mincount': 654,
5476         'info_dict': {
5477             'title': '2018 Chinese New Singles (11/6 updated)',
5478             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5479             'uploader': 'LBK',
5480             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5481             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5482             'channel': 'LBK',
5483             'view_count': int,
5484             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5485             'tags': [],
5486             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5487             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5488             'modified_date': r're:\d{8}',
5489         },
5490         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5491     }, {
5492         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5493         'only_matching': True,
5494     }, {
5495         # music album playlist
5496         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5497         'only_matching': True,
5498     }]
5499
5500     @classmethod
5501     def suitable(cls, url):
5502         if YoutubeTabIE.suitable(url):
5503             return False
5504         from ..utils import parse_qs
5505         qs = parse_qs(url)
5506         if qs.get('v', [None])[0]:
5507             return False
5508         return super().suitable(url)
5509
5510     def _real_extract(self, url):
5511         playlist_id = self._match_id(url)
5512         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5513         url = update_url_query(
5514             'https://www.youtube.com/playlist',
5515             parse_qs(url) or {'list': playlist_id})
5516         if is_music_url:
5517             url = smuggle_url(url, {'is_music_url': True})
5518         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5519
5520
5521 class YoutubeYtBeIE(InfoExtractor):
5522     IE_DESC = 'youtu.be'
5523     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5524     _TESTS = [{
5525         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5526         'info_dict': {
5527             'id': 'yeWKywCrFtk',
5528             'ext': 'mp4',
5529             'title': 'Small Scale Baler and Braiding Rugs',
5530             'uploader': 'Backus-Page House Museum',
5531             'uploader_id': 'backuspagemuseum',
5532             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5533             'upload_date': '20161008',
5534             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5535             'categories': ['Nonprofits & Activism'],
5536             'tags': list,
5537             'like_count': int,
5538             'age_limit': 0,
5539             'playable_in_embed': True,
5540             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5541             'channel': 'Backus-Page House Museum',
5542             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5543             'live_status': 'not_live',
5544             'view_count': int,
5545             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5546             'availability': 'public',
5547             'duration': 59,
5548         },
5549         'params': {
5550             'noplaylist': True,
5551             'skip_download': True,
5552         },
5553     }, {
5554         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5555         'only_matching': True,
5556     }]
5557
5558     def _real_extract(self, url):
5559         mobj = self._match_valid_url(url)
5560         video_id = mobj.group('id')
5561         playlist_id = mobj.group('playlist_id')
5562         return self.url_result(
5563             update_url_query('https://www.youtube.com/watch', {
5564                 'v': video_id,
5565                 'list': playlist_id,
5566                 'feature': 'youtu.be',
5567             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5568
5569
5570 class YoutubeLivestreamEmbedIE(InfoExtractor):
5571     IE_DESC = 'YouTube livestream embeds'
5572     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5573     _TESTS = [{
5574         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5575         'only_matching': True,
5576     }]
5577
5578     def _real_extract(self, url):
5579         channel_id = self._match_id(url)
5580         return self.url_result(
5581             f'https://www.youtube.com/channel/{channel_id}/live',
5582             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5583
5584
5585 class YoutubeYtUserIE(InfoExtractor):
5586     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5587     IE_NAME = 'youtube:user'
5588     _VALID_URL = r'ytuser:(?P<id>.+)'
5589     _TESTS = [{
5590         'url': 'ytuser:phihag',
5591         'only_matching': True,
5592     }]
5593
5594     def _real_extract(self, url):
5595         user_id = self._match_id(url)
5596         return self.url_result(
5597             'https://www.youtube.com/user/%s/videos' % user_id,
5598             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5599
5600
5601 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5602     IE_NAME = 'youtube:favorites'
5603     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5604     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5605     _LOGIN_REQUIRED = True
5606     _TESTS = [{
5607         'url': ':ytfav',
5608         'only_matching': True,
5609     }, {
5610         'url': ':ytfavorites',
5611         'only_matching': True,
5612     }]
5613
5614     def _real_extract(self, url):
5615         return self.url_result(
5616             'https://www.youtube.com/playlist?list=LL',
5617             ie=YoutubeTabIE.ie_key())
5618
5619
5620 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5621     IE_NAME = 'youtube:notif'
5622     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5623     _VALID_URL = r':ytnotif(?:ication)?s?'
5624     _LOGIN_REQUIRED = True
5625     _TESTS = [{
5626         'url': ':ytnotif',
5627         'only_matching': True,
5628     }, {
5629         'url': ':ytnotifications',
5630         'only_matching': True,
5631     }]
5632
5633     def _extract_notification_menu(self, response, continuation_list):
5634         notification_list = traverse_obj(
5635             response,
5636             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5637             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5638             expected_type=list) or []
5639         continuation_list[0] = None
5640         for item in notification_list:
5641             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5642             if entry:
5643                 yield entry
5644             continuation = item.get('continuationItemRenderer')
5645             if continuation:
5646                 continuation_list[0] = continuation
5647
5648     def _extract_notification_renderer(self, notification):
5649         video_id = traverse_obj(
5650             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5651         url = f'https://www.youtube.com/watch?v={video_id}'
5652         channel_id = None
5653         if not video_id:
5654             browse_ep = traverse_obj(
5655                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5656             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5657             post_id = self._search_regex(
5658                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5659                 'post id', default=None)
5660             if not channel_id or not post_id:
5661                 return
5662             # The direct /post url redirects to this in the browser
5663             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5664
5665         channel = traverse_obj(
5666             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5667             expected_type=str)
5668         notification_title = self._get_text(notification, 'shortMessage')
5669         if notification_title:
5670             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
5671         # TODO: handle recommended videos
5672         title = self._search_regex(
5673             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
5674             'video title', default=None)
5675         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5676                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5677                        else None)
5678         return {
5679             '_type': 'url',
5680             'url': url,
5681             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5682             'video_id': video_id,
5683             'title': title,
5684             'channel_id': channel_id,
5685             'channel': channel,
5686             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5687             'upload_date': upload_date,
5688         }
5689
5690     def _notification_menu_entries(self, ytcfg):
5691         continuation_list = [None]
5692         response = None
5693         for page in itertools.count(1):
5694             ctoken = traverse_obj(
5695                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5696             response = self._extract_response(
5697                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5698                 ep='notification/get_notification_menu', check_get_keys='actions',
5699                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5700             yield from self._extract_notification_menu(response, continuation_list)
5701             if not continuation_list[0]:
5702                 break
5703
5704     def _real_extract(self, url):
5705         display_id = 'notifications'
5706         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5707         self._report_playlist_authcheck(ytcfg)
5708         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5709
5710
5711 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5712     IE_DESC = 'YouTube search'
5713     IE_NAME = 'youtube:search'
5714     _SEARCH_KEY = 'ytsearch'
5715     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5716     _TESTS = [{
5717         'url': 'ytsearch5:youtube-dl test video',
5718         'playlist_count': 5,
5719         'info_dict': {
5720             'id': 'youtube-dl test video',
5721             'title': 'youtube-dl test video',
5722         }
5723     }]
5724
5725
5726 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5727     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5728     _SEARCH_KEY = 'ytsearchdate'
5729     IE_DESC = 'YouTube search, newest videos first'
5730     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5731     _TESTS = [{
5732         'url': 'ytsearchdate5:youtube-dl test video',
5733         'playlist_count': 5,
5734         'info_dict': {
5735             'id': 'youtube-dl test video',
5736             'title': 'youtube-dl test video',
5737         }
5738     }]
5739
5740
5741 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5742     IE_DESC = 'YouTube search URLs with sorting and filter support'
5743     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5744     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5745     _TESTS = [{
5746         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5747         'playlist_mincount': 5,
5748         'info_dict': {
5749             'id': 'youtube-dl test video',
5750             'title': 'youtube-dl test video',
5751         }
5752     }, {
5753         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5754         'playlist_mincount': 5,
5755         'info_dict': {
5756             'id': 'python',
5757             'title': 'python',
5758         }
5759     }, {
5760         'url': 'https://www.youtube.com/results?search_query=%23cats',
5761         'playlist_mincount': 1,
5762         'info_dict': {
5763             'id': '#cats',
5764             'title': '#cats',
5765             'entries': [{
5766                 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5767                 'title': '#cats',
5768             }],
5769         },
5770     }, {
5771         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5772         'only_matching': True,
5773     }]
5774
5775     def _real_extract(self, url):
5776         qs = parse_qs(url)
5777         query = (qs.get('search_query') or qs.get('q'))[0]
5778         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5779
5780
5781 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5782     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5783     IE_NAME = 'youtube:music:search_url'
5784     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5785     _TESTS = [{
5786         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5787         'playlist_count': 16,
5788         'info_dict': {
5789             'id': 'royalty free music',
5790             'title': 'royalty free music',
5791         }
5792     }, {
5793         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5794         'playlist_mincount': 30,
5795         'info_dict': {
5796             'id': 'royalty free music - songs',
5797             'title': 'royalty free music - songs',
5798         },
5799         'params': {'extract_flat': 'in_playlist'}
5800     }, {
5801         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5802         'playlist_mincount': 30,
5803         'info_dict': {
5804             'id': 'royalty free music - community playlists',
5805             'title': 'royalty free music - community playlists',
5806         },
5807         'params': {'extract_flat': 'in_playlist'}
5808     }]
5809
5810     _SECTIONS = {
5811         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5812         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5813         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5814         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5815         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5816         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5817     }
5818
5819     def _real_extract(self, url):
5820         qs = parse_qs(url)
5821         query = (qs.get('search_query') or qs.get('q'))[0]
5822         params = qs.get('sp', (None,))[0]
5823         if params:
5824             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5825         else:
5826             section = compat_urllib_parse_unquote_plus((url.split('#') + [''])[1]).lower()
5827             params = self._SECTIONS.get(section)
5828             if not params:
5829                 section = None
5830         title = join_nonempty(query, section, delim=' - ')
5831         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5832
5833
5834 class YoutubeFeedsInfoExtractor(InfoExtractor):
5835     """
5836     Base class for feed extractors
5837     Subclasses must re-define the _FEED_NAME property.
5838     """
5839     _LOGIN_REQUIRED = True
5840     _FEED_NAME = 'feeds'
5841
5842     def _real_initialize(self):
5843         YoutubeBaseInfoExtractor._check_login_required(self)
5844
5845     @classproperty
5846     def IE_NAME(self):
5847         return f'youtube:{self._FEED_NAME}'
5848
5849     def _real_extract(self, url):
5850         return self.url_result(
5851             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5852
5853
5854 class YoutubeWatchLaterIE(InfoExtractor):
5855     IE_NAME = 'youtube:watchlater'
5856     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5857     _VALID_URL = r':ytwatchlater'
5858     _TESTS = [{
5859         'url': ':ytwatchlater',
5860         'only_matching': True,
5861     }]
5862
5863     def _real_extract(self, url):
5864         return self.url_result(
5865             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5866
5867
5868 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5869     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5870     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5871     _FEED_NAME = 'recommended'
5872     _LOGIN_REQUIRED = False
5873     _TESTS = [{
5874         'url': ':ytrec',
5875         'only_matching': True,
5876     }, {
5877         'url': ':ytrecommended',
5878         'only_matching': True,
5879     }, {
5880         'url': 'https://youtube.com',
5881         'only_matching': True,
5882     }]
5883
5884
5885 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5886     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5887     _VALID_URL = r':ytsub(?:scription)?s?'
5888     _FEED_NAME = 'subscriptions'
5889     _TESTS = [{
5890         'url': ':ytsubs',
5891         'only_matching': True,
5892     }, {
5893         'url': ':ytsubscriptions',
5894         'only_matching': True,
5895     }]
5896
5897
5898 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5899     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5900     _VALID_URL = r':ythis(?:tory)?'
5901     _FEED_NAME = 'history'
5902     _TESTS = [{
5903         'url': ':ythistory',
5904         'only_matching': True,
5905     }]
5906
5907
5908 class YoutubeStoriesIE(InfoExtractor):
5909     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5910     IE_NAME = 'youtube:stories'
5911     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5912     _TESTS = [{
5913         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5914         'only_matching': True,
5915     }]
5916
5917     def _real_extract(self, url):
5918         playlist_id = f'RLTD{self._match_id(url)}'
5919         return self.url_result(
5920             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5921             ie=YoutubeTabIE, video_id=playlist_id)
5922
5923
5924 class YoutubeTruncatedURLIE(InfoExtractor):
5925     IE_NAME = 'youtube:truncated_url'
5926     IE_DESC = False  # Do not list
5927     _VALID_URL = r'''(?x)
5928         (?:https?://)?
5929         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5930         (?:watch\?(?:
5931             feature=[a-z_]+|
5932             annotation_id=annotation_[^&]+|
5933             x-yt-cl=[0-9]+|
5934             hl=[^&]*|
5935             t=[0-9]+
5936         )?
5937         |
5938             attribution_link\?a=[^&]+
5939         )
5940         $
5941     '''
5942
5943     _TESTS = [{
5944         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5945         'only_matching': True,
5946     }, {
5947         'url': 'https://www.youtube.com/watch?',
5948         'only_matching': True,
5949     }, {
5950         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5951         'only_matching': True,
5952     }, {
5953         'url': 'https://www.youtube.com/watch?feature=foo',
5954         'only_matching': True,
5955     }, {
5956         'url': 'https://www.youtube.com/watch?hl=en-GB',
5957         'only_matching': True,
5958     }, {
5959         'url': 'https://www.youtube.com/watch?t=2372',
5960         'only_matching': True,
5961     }]
5962
5963     def _real_extract(self, url):
5964         raise ExtractorError(
5965             'Did you forget to quote the URL? Remember that & is a meta '
5966             'character in most shells, so you want to put the URL in quotes, '
5967             'like  youtube-dl '
5968             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5969             ' or simply  youtube-dl BaW_jenozKc  .',
5970             expected=True)
5971
5972
5973 class YoutubeClipIE(InfoExtractor):
5974     IE_NAME = 'youtube:clip'
5975     IE_DESC = False  # Do not list
5976     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/'
5977
5978     def _real_extract(self, url):
5979         self.report_warning('YouTube clips are not currently supported. The entire video will be downloaded instead')
5980         return self.url_result(url, 'Generic')
5981
5982
5983 class YoutubeTruncatedIDIE(InfoExtractor):
5984     IE_NAME = 'youtube:truncated_id'
5985     IE_DESC = False  # Do not list
5986     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
5987
5988     _TESTS = [{
5989         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
5990         'only_matching': True,
5991     }]
5992
5993     def _real_extract(self, url):
5994         video_id = self._match_id(url)
5995         raise ExtractorError(
5996             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
5997             expected=True)