yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16 import urllib.error
  17 import urllib.parse
  18
  19 from .common import InfoExtractor, SearchInfoExtractor
  20 from ..compat import functools
  21 from ..jsinterp import JSInterpreter
  22 from ..utils import (
  23     NO_DEFAULT,
  24     ExtractorError,
  25     UserNotLive,
  26     bug_reports_message,
  27     classproperty,
  28     clean_html,
  29     datetime_from_str,
  30     dict_get,
  31     error_to_compat_str,
  32     float_or_none,
  33     format_field,
  34     get_first,
  35     int_or_none,
  36     is_html,
  37     join_nonempty,
  38     js_to_json,
  39     mimetype2ext,
  40     network_exceptions,
  41     orderedSet,
  42     parse_codecs,
  43     parse_count,
  44     parse_duration,
  45     parse_iso8601,
  46     parse_qs,
  47     qualities,
  48     remove_end,
  49     remove_start,
  50     smuggle_url,
  51     str_or_none,
  52     str_to_int,
  53     strftime_or_none,
  54     traverse_obj,
  55     try_get,
  56     unescapeHTML,
  57     unified_strdate,
  58     unified_timestamp,
  59     unsmuggle_url,
  60     update_url_query,
  61     url_or_none,
  62     urljoin,
  63     variadic,
  64 )
  65
  66 # any clients starting with _ cannot be explicitly requested by the user
  67 INNERTUBE_CLIENTS = {
  68     'web': {
  69         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  70         'INNERTUBE_CONTEXT': {
  71             'client': {
  72                 'clientName': 'WEB',
  73                 'clientVersion': '2.20211221.00.00',
  74             }
  75         },
  76         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  77     },
  78     'web_embedded': {
  79         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  80         'INNERTUBE_CONTEXT': {
  81             'client': {
  82                 'clientName': 'WEB_EMBEDDED_PLAYER',
  83                 'clientVersion': '1.20211215.00.01',
  84             },
  85         },
  86         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  87     },
  88     'web_music': {
  89         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  90         'INNERTUBE_HOST': 'music.youtube.com',
  91         'INNERTUBE_CONTEXT': {
  92             'client': {
  93                 'clientName': 'WEB_REMIX',
  94                 'clientVersion': '1.20211213.00.00',
  95             }
  96         },
  97         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
  98     },
  99     'web_creator': {
 100         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 101         'INNERTUBE_CONTEXT': {
 102             'client': {
 103                 'clientName': 'WEB_CREATOR',
 104                 'clientVersion': '1.20211220.02.00',
 105             }
 106         },
 107         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 108     },
 109     'android': {
 110         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 111         'INNERTUBE_CONTEXT': {
 112             'client': {
 113                 'clientName': 'ANDROID',
 114                 'clientVersion': '16.49',
 115             }
 116         },
 117         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 118         'REQUIRE_JS_PLAYER': False
 119     },
 120     'android_embedded': {
 121         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 122         'INNERTUBE_CONTEXT': {
 123             'client': {
 124                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 125                 'clientVersion': '16.49',
 126             },
 127         },
 128         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 129         'REQUIRE_JS_PLAYER': False
 130     },
 131     'android_music': {
 132         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 133         'INNERTUBE_CONTEXT': {
 134             'client': {
 135                 'clientName': 'ANDROID_MUSIC',
 136                 'clientVersion': '4.57',
 137             }
 138         },
 139         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 140         'REQUIRE_JS_PLAYER': False
 141     },
 142     'android_creator': {
 143         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 144         'INNERTUBE_CONTEXT': {
 145             'client': {
 146                 'clientName': 'ANDROID_CREATOR',
 147                 'clientVersion': '21.47',
 148             },
 149         },
 150         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 151         'REQUIRE_JS_PLAYER': False
 152     },
 153     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 154     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 155     'ios': {
 156         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 157         'INNERTUBE_CONTEXT': {
 158             'client': {
 159                 'clientName': 'IOS',
 160                 'clientVersion': '16.46',
 161                 'deviceModel': 'iPhone14,3',
 162             }
 163         },
 164         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 165         'REQUIRE_JS_PLAYER': False
 166     },
 167     'ios_embedded': {
 168         'INNERTUBE_CONTEXT': {
 169             'client': {
 170                 'clientName': 'IOS_MESSAGES_EXTENSION',
 171                 'clientVersion': '16.46',
 172                 'deviceModel': 'iPhone14,3',
 173             },
 174         },
 175         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 176         'REQUIRE_JS_PLAYER': False
 177     },
 178     'ios_music': {
 179         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 180         'INNERTUBE_CONTEXT': {
 181             'client': {
 182                 'clientName': 'IOS_MUSIC',
 183                 'clientVersion': '4.57',
 184             },
 185         },
 186         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 187         'REQUIRE_JS_PLAYER': False
 188     },
 189     'ios_creator': {
 190         'INNERTUBE_CONTEXT': {
 191             'client': {
 192                 'clientName': 'IOS_CREATOR',
 193                 'clientVersion': '21.47',
 194             },
 195         },
 196         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 197         'REQUIRE_JS_PLAYER': False
 198     },
 199     # mweb has 'ultralow' formats
 200     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 201     'mweb': {
 202         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 203         'INNERTUBE_CONTEXT': {
 204             'client': {
 205                 'clientName': 'MWEB',
 206                 'clientVersion': '2.20211221.01.00',
 207             }
 208         },
 209         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 210     },
 211     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 212     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 213     'tv_embedded': {
 214         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 215         'INNERTUBE_CONTEXT': {
 216             'client': {
 217                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 218                 'clientVersion': '2.0',
 219             },
 220         },
 221         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 222     },
 223 }
 224
 225
 226 def _split_innertube_client(client_name):
 227     variant, *base = client_name.rsplit('.', 1)
 228     if base:
 229         return variant, base[0], variant
 230     base, *variant = client_name.split('_', 1)
 231     return client_name, base, variant[0] if variant else None
 232
 233
 234 def build_innertube_clients():
 235     THIRD_PARTY = {
 236         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 237     }
 238     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 239     priority = qualities(BASE_CLIENTS[::-1])
 240
 241     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 242         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 243         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 244         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 245         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 246
 247         _, base_client, variant = _split_innertube_client(client)
 248         ytcfg['priority'] = 10 * priority(base_client)
 249
 250         if not variant:
 251             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 252             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 253             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 254             embedscreen['priority'] -= 3
 255         elif variant == 'embedded':
 256             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 257             ytcfg['priority'] -= 2
 258         else:
 259             ytcfg['priority'] -= 3
 260
 261
 262 build_innertube_clients()
 263
 264
 265 class YoutubeBaseInfoExtractor(InfoExtractor):
 266     """Provide base functions for Youtube extractors"""
 267
 268     _RESERVED_NAMES = (
 269         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 270         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 271         r'browse|oembed|get_video_info|iframe_api|s/player|'
 272         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 273
 274     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 275
 276     # _NETRC_MACHINE = 'youtube'
 277
 278     # If True it will raise an error if no login info is provided
 279     _LOGIN_REQUIRED = False
 280
 281     _INVIDIOUS_SITES = (
 282         # invidious-redirect websites
 283         r'(?:www\.)?redirect\.invidious\.io',
 284         r'(?:(?:www|dev)\.)?invidio\.us',
 285         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 286         r'(?:www\.)?invidious\.pussthecat\.org',
 287         r'(?:www\.)?invidious\.zee\.li',
 288         r'(?:www\.)?invidious\.ethibox\.fr',
 289         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 290         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 291         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 292         # youtube-dl invidious instances list
 293         r'(?:(?:www|no)\.)?invidiou\.sh',
 294         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 295         r'(?:www\.)?invidious\.kabi\.tk',
 296         r'(?:www\.)?invidious\.mastodon\.host',
 297         r'(?:www\.)?invidious\.zapashcanon\.fr',
 298         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 299         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 300         r'(?:www\.)?invidious\.himiko\.cloud',
 301         r'(?:www\.)?invidious\.reallyancient\.tech',
 302         r'(?:www\.)?invidious\.tube',
 303         r'(?:www\.)?invidiou\.site',
 304         r'(?:www\.)?invidious\.site',
 305         r'(?:www\.)?invidious\.xyz',
 306         r'(?:www\.)?invidious\.nixnet\.xyz',
 307         r'(?:www\.)?invidious\.048596\.xyz',
 308         r'(?:www\.)?invidious\.drycat\.fr',
 309         r'(?:www\.)?inv\.skyn3t\.in',
 310         r'(?:www\.)?tube\.poal\.co',
 311         r'(?:www\.)?tube\.connect\.cafe',
 312         r'(?:www\.)?vid\.wxzm\.sx',
 313         r'(?:www\.)?vid\.mint\.lgbt',
 314         r'(?:www\.)?vid\.puffyan\.us',
 315         r'(?:www\.)?yewtu\.be',
 316         r'(?:www\.)?yt\.elukerio\.org',
 317         r'(?:www\.)?yt\.lelux\.fi',
 318         r'(?:www\.)?invidious\.ggc-project\.de',
 319         r'(?:www\.)?yt\.maisputain\.ovh',
 320         r'(?:www\.)?ytprivate\.com',
 321         r'(?:www\.)?invidious\.13ad\.de',
 322         r'(?:www\.)?invidious\.toot\.koeln',
 323         r'(?:www\.)?invidious\.fdn\.fr',
 324         r'(?:www\.)?watch\.nettohikari\.com',
 325         r'(?:www\.)?invidious\.namazso\.eu',
 326         r'(?:www\.)?invidious\.silkky\.cloud',
 327         r'(?:www\.)?invidious\.exonip\.de',
 328         r'(?:www\.)?invidious\.riverside\.rocks',
 329         r'(?:www\.)?invidious\.blamefran\.net',
 330         r'(?:www\.)?invidious\.moomoo\.de',
 331         r'(?:www\.)?ytb\.trom\.tf',
 332         r'(?:www\.)?yt\.cyberhost\.uk',
 333         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 334         r'(?:www\.)?qklhadlycap4cnod\.onion',
 335         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 336         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 337         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 338         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 339         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 340         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 341         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 342         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 343         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 344         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 345         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 346         r'(?:www\.)?piped\.kavin\.rocks',
 347         r'(?:www\.)?piped\.silkky\.cloud',
 348         r'(?:www\.)?piped\.tokhmi\.xyz',
 349         r'(?:www\.)?piped\.moomoo\.me',
 350         r'(?:www\.)?il\.ax',
 351         r'(?:www\.)?piped\.syncpundit\.com',
 352         r'(?:www\.)?piped\.mha\.fi',
 353         r'(?:www\.)?piped\.mint\.lgbt',
 354         r'(?:www\.)?piped\.privacy\.com\.de',
 355     )
 356
 357     def _initialize_consent(self):
 358         cookies = self._get_cookies('https://www.youtube.com/')
 359         if cookies.get('__Secure-3PSID'):
 360             return
 361         consent_id = None
 362         consent = cookies.get('CONSENT')
 363         if consent:
 364             if 'YES' in consent.value:
 365                 return
 366             consent_id = self._search_regex(
 367                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 368         if not consent_id:
 369             consent_id = random.randint(100, 999)
 370         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 371
 372     def _initialize_pref(self):
 373         cookies = self._get_cookies('https://www.youtube.com/')
 374         pref_cookie = cookies.get('PREF')
 375         pref = {}
 376         if pref_cookie:
 377             try:
 378                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 379             except ValueError:
 380                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 381         pref.update({'hl': 'en', 'tz': 'UTC'})
 382         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 383
 384     def _real_initialize(self):
 385         self._initialize_pref()
 386         self._initialize_consent()
 387         self._check_login_required()
 388
 389     def _check_login_required(self):
 390         if self._LOGIN_REQUIRED and not self._cookies_passed:
 391             self.raise_login_required('Login details are needed to download this content', method='cookies')
 392
 393     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 394     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 395
 396     def _get_default_ytcfg(self, client='web'):
 397         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 398
 399     def _get_innertube_host(self, client='web'):
 400         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 401
 402     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 403         # try_get but with fallback to default ytcfg client values when present
 404         _func = lambda y: try_get(y, getter, expected_type)
 405         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 406
 407     def _extract_client_name(self, ytcfg, default_client='web'):
 408         return self._ytcfg_get_safe(
 409             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 410                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 411
 412     def _extract_client_version(self, ytcfg, default_client='web'):
 413         return self._ytcfg_get_safe(
 414             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 415                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 416
 417     def _select_api_hostname(self, req_api_hostname, default_client=None):
 418         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 419                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 420
 421     def _extract_api_key(self, ytcfg=None, default_client='web'):
 422         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 423
 424     def _extract_context(self, ytcfg=None, default_client='web'):
 425         context = get_first(
 426             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 427         # Enforce language and tz for extraction
 428         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 429         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 430         return context
 431
 432     _SAPISID = None
 433
 434     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 435         time_now = round(time.time())
 436         if self._SAPISID is None:
 437             yt_cookies = self._get_cookies('https://www.youtube.com')
 438             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 439             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 440             sapisid_cookie = dict_get(
 441                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 442             if sapisid_cookie and sapisid_cookie.value:
 443                 self._SAPISID = sapisid_cookie.value
 444                 self.write_debug('Extracted SAPISID cookie')
 445                 # SAPISID cookie is required if not already present
 446                 if not yt_cookies.get('SAPISID'):
 447                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 448                     self._set_cookie(
 449                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 450             else:
 451                 self._SAPISID = False
 452         if not self._SAPISID:
 453             return None
 454         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 455         sapisidhash = hashlib.sha1(
 456             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 457         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 458
 459     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 460                   note='Downloading API JSON', errnote='Unable to download API page',
 461                   context=None, api_key=None, api_hostname=None, default_client='web'):
 462
 463         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 464         data.update(query)
 465         real_headers = self.generate_api_headers(default_client=default_client)
 466         real_headers.update({'content-type': 'application/json'})
 467         if headers:
 468             real_headers.update(headers)
 469         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 470                    or api_key or self._extract_api_key(default_client=default_client))
 471         return self._download_json(
 472             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 473             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 474             data=json.dumps(data).encode('utf8'), headers=real_headers,
 475             query={'key': api_key, 'prettyPrint': 'false'})
 476
 477     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 478         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 479
 480     @staticmethod
 481     def _extract_session_index(*data):
 482         """
 483         Index of current account in account list.
 484         See: https://github.com/yt-dlp/yt-dlp/pull/519
 485         """
 486         for ytcfg in data:
 487             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 488             if session_index is not None:
 489                 return session_index
 490
 491     # Deprecated?
 492     def _extract_identity_token(self, ytcfg=None, webpage=None):
 493         if ytcfg:
 494             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 495             if token:
 496                 return token
 497         if webpage:
 498             return self._search_regex(
 499                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 500                 'identity token', default=None, fatal=False)
 501
 502     @staticmethod
 503     def _extract_account_syncid(*args):
 504         """
 505         Extract syncId required to download private playlists of secondary channels
 506         @params response and/or ytcfg
 507         """
 508         for data in args:
 509             # ytcfg includes channel_syncid if on secondary channel
 510             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 511             if delegated_sid:
 512                 return delegated_sid
 513             sync_ids = (try_get(
 514                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 515                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 516             if len(sync_ids) >= 2 and sync_ids[1]:
 517                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 518                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 519                 return sync_ids[0]
 520
 521     @staticmethod
 522     def _extract_visitor_data(*args):
 523         """
 524         Extracts visitorData from an API response or ytcfg
 525         Appears to be used to track session state
 526         """
 527         return get_first(
 528             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 529             expected_type=str)
 530
 531     @functools.cached_property
 532     def is_authenticated(self):
 533         return bool(self._generate_sapisidhash_header())
 534
 535     def extract_ytcfg(self, video_id, webpage):
 536         if not webpage:
 537             return {}
 538         return self._parse_json(
 539             self._search_regex(
 540                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 541                 default='{}'), video_id, fatal=False) or {}
 542
 543     def generate_api_headers(
 544             self, *, ytcfg=None, account_syncid=None, session_index=None,
 545             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 546
 547         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 548         headers = {
 549             'X-YouTube-Client-Name': str(
 550                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 551             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 552             'Origin': origin,
 553             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 554             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 555             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 556         }
 557         if session_index is None:
 558             session_index = self._extract_session_index(ytcfg)
 559         if account_syncid or session_index is not None:
 560             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 561
 562         auth = self._generate_sapisidhash_header(origin)
 563         if auth is not None:
 564             headers['Authorization'] = auth
 565             headers['X-Origin'] = origin
 566         return {h: v for h, v in headers.items() if v is not None}
 567
 568     def _download_ytcfg(self, client, video_id):
 569         url = {
 570             'web': 'https://www.youtube.com',
 571             'web_music': 'https://music.youtube.com',
 572             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 573         }.get(client)
 574         if not url:
 575             return {}
 576         webpage = self._download_webpage(
 577             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 578         return self.extract_ytcfg(video_id, webpage) or {}
 579
 580     @staticmethod
 581     def _build_api_continuation_query(continuation, ctp=None):
 582         query = {
 583             'continuation': continuation
 584         }
 585         # TODO: Inconsistency with clickTrackingParams.
 586         # Currently we have a fixed ctp contained within context (from ytcfg)
 587         # and a ctp in root query for continuation.
 588         if ctp:
 589             query['clickTracking'] = {'clickTrackingParams': ctp}
 590         return query
 591
 592     @classmethod
 593     def _extract_next_continuation_data(cls, renderer):
 594         next_continuation = try_get(
 595             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 596                        lambda x: x['continuation']['reloadContinuationData']), dict)
 597         if not next_continuation:
 598             return
 599         continuation = next_continuation.get('continuation')
 600         if not continuation:
 601             return
 602         ctp = next_continuation.get('clickTrackingParams')
 603         return cls._build_api_continuation_query(continuation, ctp)
 604
 605     @classmethod
 606     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 607         if isinstance(continuation_ep, dict):
 608             continuation = try_get(
 609                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 610             if not continuation:
 611                 return
 612             ctp = continuation_ep.get('clickTrackingParams')
 613             return cls._build_api_continuation_query(continuation, ctp)
 614
 615     @classmethod
 616     def _extract_continuation(cls, renderer):
 617         next_continuation = cls._extract_next_continuation_data(renderer)
 618         if next_continuation:
 619             return next_continuation
 620
 621         contents = []
 622         for key in ('contents', 'items'):
 623             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 624
 625         for content in contents:
 626             if not isinstance(content, dict):
 627                 continue
 628             continuation_ep = try_get(
 629                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 630                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 631                 dict)
 632             continuation = cls._extract_continuation_ep_data(continuation_ep)
 633             if continuation:
 634                 return continuation
 635
 636     @classmethod
 637     def _extract_alerts(cls, data):
 638         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 639             if not isinstance(alert_dict, dict):
 640                 continue
 641             for alert in alert_dict.values():
 642                 alert_type = alert.get('type')
 643                 if not alert_type:
 644                     continue
 645                 message = cls._get_text(alert, 'text')
 646                 if message:
 647                     yield alert_type, message
 648
 649     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 650         errors = []
 651         warnings = []
 652         for alert_type, alert_message in alerts:
 653             if alert_type.lower() == 'error' and fatal:
 654                 errors.append([alert_type, alert_message])
 655             else:
 656                 warnings.append([alert_type, alert_message])
 657
 658         for alert_type, alert_message in (warnings + errors[:-1]):
 659             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 660         if errors:
 661             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 662
 663     def _extract_and_report_alerts(self, data, *args, **kwargs):
 664         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 665
 666     def _extract_badges(self, renderer: dict):
 667         badges = set()
 668         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 669             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], str)
 670             if label:
 671                 badges.add(label.lower())
 672         return badges
 673
 674     @staticmethod
 675     def _get_text(data, *path_list, max_runs=None):
 676         for path in path_list or [None]:
 677             if path is None:
 678                 obj = [data]
 679             else:
 680                 obj = traverse_obj(data, path, default=[])
 681                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 682                     obj = [obj]
 683             for item in obj:
 684                 text = try_get(item, lambda x: x['simpleText'], str)
 685                 if text:
 686                     return text
 687                 runs = try_get(item, lambda x: x['runs'], list) or []
 688                 if not runs and isinstance(item, list):
 689                     runs = item
 690
 691                 runs = runs[:min(len(runs), max_runs or len(runs))]
 692                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 693                 if text:
 694                     return text
 695
 696     def _get_count(self, data, *path_list):
 697         count_text = self._get_text(data, *path_list) or ''
 698         count = parse_count(count_text)
 699         if count is None:
 700             count = str_to_int(
 701                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 702         return count
 703
 704     @staticmethod
 705     def _extract_thumbnails(data, *path_list):
 706         """
 707         Extract thumbnails from thumbnails dict
 708         @param path_list: path list to level that contains 'thumbnails' key
 709         """
 710         thumbnails = []
 711         for path in path_list or [()]:
 712             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 713                 thumbnail_url = url_or_none(thumbnail.get('url'))
 714                 if not thumbnail_url:
 715                     continue
 716                 # Sometimes youtube gives a wrong thumbnail URL. See:
 717                 # https://github.com/yt-dlp/yt-dlp/issues/233
 718                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 719                 if 'maxresdefault' in thumbnail_url:
 720                     thumbnail_url = thumbnail_url.split('?')[0]
 721                 thumbnails.append({
 722                     'url': thumbnail_url,
 723                     'height': int_or_none(thumbnail.get('height')),
 724                     'width': int_or_none(thumbnail.get('width')),
 725                 })
 726         return thumbnails
 727
 728     @staticmethod
 729     def extract_relative_time(relative_time_text):
 730         """
 731         Extracts a relative time from string and converts to dt object
 732         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 733         """
 734         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 735         if mobj:
 736             start = mobj.group('start')
 737             if start:
 738                 return datetime_from_str(start)
 739             try:
 740                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 741             except ValueError:
 742                 return None
 743
 744     def _extract_time_text(self, renderer, *path_list):
 745         """@returns (timestamp, time_text)"""
 746         text = self._get_text(renderer, *path_list) or ''
 747         dt = self.extract_relative_time(text)
 748         timestamp = None
 749         if isinstance(dt, datetime.datetime):
 750             timestamp = calendar.timegm(dt.timetuple())
 751
 752         if timestamp is None:
 753             timestamp = (
 754                 unified_timestamp(text) or unified_timestamp(
 755                     self._search_regex(
 756                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 757                         text.lower(), 'time text', default=None)))
 758
 759         if text and timestamp is None:
 760             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 761         return timestamp, text
 762
 763     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 764                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 765                           default_client='web'):
 766         response = None
 767         last_error = None
 768         count = -1
 769         retries = self.get_param('extractor_retries', 3)
 770         if check_get_keys is None:
 771             check_get_keys = []
 772         while count < retries:
 773             count += 1
 774             if last_error:
 775                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 776             try:
 777                 response = self._call_api(
 778                     ep=ep, fatal=True, headers=headers,
 779                     video_id=item_id, query=query,
 780                     context=self._extract_context(ytcfg, default_client),
 781                     api_key=self._extract_api_key(ytcfg, default_client),
 782                     api_hostname=api_hostname, default_client=default_client,
 783                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 784             except ExtractorError as e:
 785                 if isinstance(e.cause, network_exceptions):
 786                     if isinstance(e.cause, urllib.error.HTTPError):
 787                         first_bytes = e.cause.read(512)
 788                         if not is_html(first_bytes):
 789                             yt_error = try_get(
 790                                 self._parse_json(
 791                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 792                                 lambda x: x['error']['message'], str)
 793                             if yt_error:
 794                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 795                     # Downloading page may result in intermittent 5xx HTTP error
 796                     # Sometimes a 404 is also received. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 797                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 798                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 799                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
 800                         last_error = error_to_compat_str(e.cause or e.msg)
 801                         if count < retries:
 802                             continue
 803                 if fatal:
 804                     raise
 805                 else:
 806                     self.report_warning(error_to_compat_str(e))
 807                     return
 808
 809             else:
 810                 try:
 811                     self._extract_and_report_alerts(response, only_once=True)
 812                 except ExtractorError as e:
 813                     # YouTube servers may return errors we want to retry on in a 200 OK response
 814                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 815                     if 'unknown error' in e.msg.lower():
 816                         last_error = e.msg
 817                         continue
 818                     if fatal:
 819                         raise
 820                     self.report_warning(error_to_compat_str(e))
 821                     return
 822                 if not check_get_keys or dict_get(response, check_get_keys):
 823                     break
 824                 # Youtube sometimes sends incomplete data
 825                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 826                 last_error = 'Incomplete data received'
 827                 if count >= retries:
 828                     if fatal:
 829                         raise ExtractorError(last_error)
 830                     else:
 831                         self.report_warning(last_error)
 832                         return
 833         return response
 834
 835     @staticmethod
 836     def is_music_url(url):
 837         return re.match(r'https?://music\.youtube\.com/', url) is not None
 838
 839     def _extract_video(self, renderer):
 840         video_id = renderer.get('videoId')
 841         title = self._get_text(renderer, 'title')
 842         description = self._get_text(renderer, 'descriptionSnippet')
 843         duration = parse_duration(self._get_text(
 844             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 845         if duration is None:
 846             duration = parse_duration(self._search_regex(
 847                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 848                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 849                 video_id, default=None, group='duration'))
 850
 851         view_count = self._get_count(renderer, 'viewCountText')
 852
 853         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 854         channel_id = traverse_obj(
 855             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 856             expected_type=str, get_all=False)
 857         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 858         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 859         overlay_style = traverse_obj(
 860             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 861             get_all=False, expected_type=str)
 862         badges = self._extract_badges(renderer)
 863         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 864         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 865             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 866             expected_type=str)) or ''
 867         url = f'https://www.youtube.com/watch?v={video_id}'
 868         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 869             url = f'https://www.youtube.com/shorts/{video_id}'
 870
 871         return {
 872             '_type': 'url',
 873             'ie_key': YoutubeIE.ie_key(),
 874             'id': video_id,
 875             'url': url,
 876             'title': title,
 877             'description': description,
 878             'duration': duration,
 879             'view_count': view_count,
 880             'uploader': uploader,
 881             'channel_id': channel_id,
 882             'thumbnails': thumbnails,
 883             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 884                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 885                             else None),
 886             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 887                             else 'was_live' if 'streamed' in time_text.lower()
 888                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 889                             else None),
 890             'release_timestamp': scheduled_timestamp,
 891             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 892         }
 893
 894
 895 class YoutubeIE(YoutubeBaseInfoExtractor):
 896     IE_DESC = 'YouTube'
 897     _VALID_URL = r"""(?x)^
 898                      (
 899                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 900                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 901                             (?:www\.)?deturl\.com/www\.youtube\.com|
 902                             (?:www\.)?pwnyoutube\.com|
 903                             (?:www\.)?hooktube\.com|
 904                             (?:www\.)?yourepeat\.com|
 905                             tube\.majestyc\.net|
 906                             %(invidious)s|
 907                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 908                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 909                          (?:                                                  # the various things that can precede the ID:
 910                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 911                              |(?:                                             # or the v= param in all its forms
 912                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 913                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 914                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 915                                  v=
 916                              )
 917                          ))
 918                          |(?:
 919                             youtu\.be|                                        # just youtu.be/xxxx
 920                             vid\.plus|                                        # or vid.plus/xxxx
 921                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 922                             %(invidious)s
 923                          )/
 924                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 925                          )
 926                      )?                                                       # all until now is optional -> you can pass the naked ID
 927                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 928                      (?(1).+)?                                                # if we found the ID, everything can follow
 929                      (?:\#|$)""" % {
 930         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 931     }
 932     _EMBED_REGEX = [r'''(?x)
 933         (?:
 934             <iframe[^>]+?src=|
 935             data-video-url=|
 936             <embed[^>]+?src=|
 937             embedSWF\(?:\s*|
 938             <object[^>]+data=|
 939             new\s+SWFObject\(
 940         )
 941         (["\'])
 942             (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
 943             (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
 944         \1''']
 945     _PLAYER_INFO_RE = (
 946         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 947         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 948         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 949     )
 950     _formats = {
 951         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 952         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 953         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 954         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 955         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 956         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 957         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 958         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 959         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 960         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 961         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 962         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 963         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 964         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 965         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 966         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 967         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 968         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 969
 970
 971         # 3D videos
 972         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 973         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 974         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 975         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 976         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 977         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 978         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 979
 980         # Apple HTTP Live Streaming
 981         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 982         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 983         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 984         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 985         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 986         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 987         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 988         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 989
 990         # DASH mp4 video
 991         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 992         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 993         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 994         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 995         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 996         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 997         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 998         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 999         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1000         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1001         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1002         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1003
1004         # Dash mp4 audio
1005         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1006         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1007         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1008         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1009         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1010         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1011         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1012
1013         # Dash webm
1014         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1015         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1016         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1017         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1018         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1019         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1020         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1021         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1022         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1023         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1024         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1025         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1026         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1027         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1028         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1029         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1030         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1031         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1032         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1033         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1034         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1035         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1036
1037         # Dash webm audio
1038         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1039         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1040
1041         # Dash webm audio with opus inside
1042         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1043         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1044         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1045
1046         # RTMP (unnamed)
1047         '_rtmp': {'protocol': 'rtmp'},
1048
1049         # av01 video only formats sometimes served with "unknown" codecs
1050         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1051         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1052         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1053         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1054         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1055         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1056         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1057         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1058     }
1059     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1060
1061     _GEO_BYPASS = False
1062
1063     IE_NAME = 'youtube'
1064     _TESTS = [
1065         {
1066             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1067             'info_dict': {
1068                 'id': 'BaW_jenozKc',
1069                 'ext': 'mp4',
1070                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1071                 'uploader': 'Philipp Hagemeister',
1072                 'uploader_id': 'phihag',
1073                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1074                 'channel': 'Philipp Hagemeister',
1075                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1076                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1077                 'upload_date': '20121002',
1078                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1079                 'categories': ['Science & Technology'],
1080                 'tags': ['youtube-dl'],
1081                 'duration': 10,
1082                 'view_count': int,
1083                 'like_count': int,
1084                 'availability': 'public',
1085                 'playable_in_embed': True,
1086                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1087                 'live_status': 'not_live',
1088                 'age_limit': 0,
1089                 'start_time': 1,
1090                 'end_time': 9,
1091                 'comment_count': int,
1092                 'channel_follower_count': int
1093             }
1094         },
1095         {
1096             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1097             'note': 'Embed-only video (#1746)',
1098             'info_dict': {
1099                 'id': 'yZIXLfi8CZQ',
1100                 'ext': 'mp4',
1101                 'upload_date': '20120608',
1102                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1103                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1104                 'uploader': 'SET India',
1105                 'uploader_id': 'setindia',
1106                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1107                 'age_limit': 18,
1108             },
1109             'skip': 'Private video',
1110         },
1111         {
1112             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1113             'note': 'Use the first video ID in the URL',
1114             'info_dict': {
1115                 'id': 'BaW_jenozKc',
1116                 'ext': 'mp4',
1117                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1118                 'uploader': 'Philipp Hagemeister',
1119                 'uploader_id': 'phihag',
1120                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1121                 'channel': 'Philipp Hagemeister',
1122                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1123                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1124                 'upload_date': '20121002',
1125                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1126                 'categories': ['Science & Technology'],
1127                 'tags': ['youtube-dl'],
1128                 'duration': 10,
1129                 'view_count': int,
1130                 'like_count': int,
1131                 'availability': 'public',
1132                 'playable_in_embed': True,
1133                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1134                 'live_status': 'not_live',
1135                 'age_limit': 0,
1136                 'comment_count': int,
1137                 'channel_follower_count': int
1138             },
1139             'params': {
1140                 'skip_download': True,
1141             },
1142         },
1143         {
1144             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1145             'note': '256k DASH audio (format 141) via DASH manifest',
1146             'info_dict': {
1147                 'id': 'a9LDPn-MO4I',
1148                 'ext': 'm4a',
1149                 'upload_date': '20121002',
1150                 'uploader_id': '8KVIDEO',
1151                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1152                 'description': '',
1153                 'uploader': '8KVIDEO',
1154                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1155             },
1156             'params': {
1157                 'youtube_include_dash_manifest': True,
1158                 'format': '141',
1159             },
1160             'skip': 'format 141 not served anymore',
1161         },
1162         # DASH manifest with encrypted signature
1163         {
1164             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1165             'info_dict': {
1166                 'id': 'IB3lcPjvWLA',
1167                 'ext': 'm4a',
1168                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1169                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1170                 'duration': 244,
1171                 'uploader': 'AfrojackVEVO',
1172                 'uploader_id': 'AfrojackVEVO',
1173                 'upload_date': '20131011',
1174                 'abr': 129.495,
1175                 'like_count': int,
1176                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1177                 'playable_in_embed': True,
1178                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1179                 'view_count': int,
1180                 'track': 'The Spark',
1181                 'live_status': 'not_live',
1182                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1183                 'channel': 'Afrojack',
1184                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1185                 'tags': 'count:19',
1186                 'availability': 'public',
1187                 'categories': ['Music'],
1188                 'age_limit': 0,
1189                 'alt_title': 'The Spark',
1190                 'channel_follower_count': int
1191             },
1192             'params': {
1193                 'youtube_include_dash_manifest': True,
1194                 'format': '141/bestaudio[ext=m4a]',
1195             },
1196         },
1197         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1198         {
1199             'note': 'Embed allowed age-gate video',
1200             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1201             'info_dict': {
1202                 'id': 'HtVdAasjOgU',
1203                 'ext': 'mp4',
1204                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1205                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1206                 'duration': 142,
1207                 'uploader': 'The Witcher',
1208                 'uploader_id': 'WitcherGame',
1209                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1210                 'upload_date': '20140605',
1211                 'age_limit': 18,
1212                 'categories': ['Gaming'],
1213                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1214                 'availability': 'needs_auth',
1215                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1216                 'like_count': int,
1217                 'channel': 'The Witcher',
1218                 'live_status': 'not_live',
1219                 'tags': 'count:17',
1220                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1221                 'playable_in_embed': True,
1222                 'view_count': int,
1223                 'channel_follower_count': int
1224             },
1225         },
1226         {
1227             'note': 'Age-gate video with embed allowed in public site',
1228             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1229             'info_dict': {
1230                 'id': 'HsUATh_Nc2U',
1231                 'ext': 'mp4',
1232                 'title': 'Godzilla 2 (Official Video)',
1233                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1234                 'upload_date': '20200408',
1235                 'uploader_id': 'FlyingKitty900',
1236                 'uploader': 'FlyingKitty',
1237                 'age_limit': 18,
1238                 'availability': 'needs_auth',
1239                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1240                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1241                 'channel': 'FlyingKitty',
1242                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1243                 'view_count': int,
1244                 'categories': ['Entertainment'],
1245                 'live_status': 'not_live',
1246                 'tags': ['Flyingkitty', 'godzilla 2'],
1247                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1248                 'like_count': int,
1249                 'duration': 177,
1250                 'playable_in_embed': True,
1251                 'channel_follower_count': int
1252             },
1253         },
1254         {
1255             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1256             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1257             'info_dict': {
1258                 'id': 'Tq92D6wQ1mg',
1259                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1260                 'ext': 'mp4',
1261                 'upload_date': '20191228',
1262                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1263                 'uploader': 'Projekt Melody',
1264                 'description': 'md5:17eccca93a786d51bc67646756894066',
1265                 'age_limit': 18,
1266                 'like_count': int,
1267                 'availability': 'needs_auth',
1268                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1269                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1270                 'view_count': int,
1271                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1272                 'channel': 'Projekt Melody',
1273                 'live_status': 'not_live',
1274                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1275                 'playable_in_embed': True,
1276                 'categories': ['Entertainment'],
1277                 'duration': 106,
1278                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1279                 'comment_count': int,
1280                 'channel_follower_count': int
1281             },
1282         },
1283         {
1284             'note': 'Non-Agegated non-embeddable video',
1285             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1286             'info_dict': {
1287                 'id': 'MeJVWBSsPAY',
1288                 'ext': 'mp4',
1289                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1290                 'uploader': 'Herr Lurik',
1291                 'uploader_id': 'st3in234',
1292                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1293                 'upload_date': '20130730',
1294                 'track': 'Such mich find mich',
1295                 'age_limit': 0,
1296                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1297                 'like_count': int,
1298                 'playable_in_embed': False,
1299                 'creator': 'OOMPH!',
1300                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1301                 'view_count': int,
1302                 'alt_title': 'Such mich find mich',
1303                 'duration': 210,
1304                 'channel': 'Herr Lurik',
1305                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1306                 'categories': ['Music'],
1307                 'availability': 'public',
1308                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1309                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1310                 'live_status': 'not_live',
1311                 'artist': 'OOMPH!',
1312                 'channel_follower_count': int
1313             },
1314         },
1315         {
1316             'note': 'Non-bypassable age-gated video',
1317             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1318             'only_matching': True,
1319         },
1320         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1321         # YouTube Red ad is not captured for creator
1322         {
1323             'url': '__2ABJjxzNo',
1324             'info_dict': {
1325                 'id': '__2ABJjxzNo',
1326                 'ext': 'mp4',
1327                 'duration': 266,
1328                 'upload_date': '20100430',
1329                 'uploader_id': 'deadmau5',
1330                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1331                 'creator': 'deadmau5',
1332                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1333                 'uploader': 'deadmau5',
1334                 'title': 'Deadmau5 - Some Chords (HD)',
1335                 'alt_title': 'Some Chords',
1336                 'availability': 'public',
1337                 'tags': 'count:14',
1338                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1339                 'view_count': int,
1340                 'live_status': 'not_live',
1341                 'channel': 'deadmau5',
1342                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1343                 'like_count': int,
1344                 'track': 'Some Chords',
1345                 'artist': 'deadmau5',
1346                 'playable_in_embed': True,
1347                 'age_limit': 0,
1348                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1349                 'categories': ['Music'],
1350                 'album': 'Some Chords',
1351                 'channel_follower_count': int
1352             },
1353             'expected_warnings': [
1354                 'DASH manifest missing',
1355             ]
1356         },
1357         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1358         {
1359             'url': 'lqQg6PlCWgI',
1360             'info_dict': {
1361                 'id': 'lqQg6PlCWgI',
1362                 'ext': 'mp4',
1363                 'duration': 6085,
1364                 'upload_date': '20150827',
1365                 'uploader_id': 'olympic',
1366                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1367                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1368                 'uploader': 'Olympics',
1369                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1370                 'like_count': int,
1371                 'release_timestamp': 1343767800,
1372                 'playable_in_embed': True,
1373                 'categories': ['Sports'],
1374                 'release_date': '20120731',
1375                 'channel': 'Olympics',
1376                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1377                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1378                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1379                 'age_limit': 0,
1380                 'availability': 'public',
1381                 'live_status': 'was_live',
1382                 'view_count': int,
1383                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1384                 'channel_follower_count': int
1385             },
1386             'params': {
1387                 'skip_download': 'requires avconv',
1388             }
1389         },
1390         # Non-square pixels
1391         {
1392             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1393             'info_dict': {
1394                 'id': '_b-2C3KPAM0',
1395                 'ext': 'mp4',
1396                 'stretched_ratio': 16 / 9.,
1397                 'duration': 85,
1398                 'upload_date': '20110310',
1399                 'uploader_id': 'AllenMeow',
1400                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1401                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1402                 'uploader': '孫ᄋᄅ',
1403                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1404                 'playable_in_embed': True,
1405                 'channel': '孫ᄋᄅ',
1406                 'age_limit': 0,
1407                 'tags': 'count:11',
1408                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1409                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1410                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1411                 'view_count': int,
1412                 'categories': ['People & Blogs'],
1413                 'like_count': int,
1414                 'live_status': 'not_live',
1415                 'availability': 'unlisted',
1416                 'comment_count': int,
1417                 'channel_follower_count': int
1418             },
1419         },
1420         # url_encoded_fmt_stream_map is empty string
1421         {
1422             'url': 'qEJwOuvDf7I',
1423             'info_dict': {
1424                 'id': 'qEJwOuvDf7I',
1425                 'ext': 'webm',
1426                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1427                 'description': '',
1428                 'upload_date': '20150404',
1429                 'uploader_id': 'spbelect',
1430                 'uploader': 'Наблюдатели Петербурга',
1431             },
1432             'params': {
1433                 'skip_download': 'requires avconv',
1434             },
1435             'skip': 'This live event has ended.',
1436         },
1437         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1438         {
1439             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1440             'info_dict': {
1441                 'id': 'FIl7x6_3R5Y',
1442                 'ext': 'webm',
1443                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1444                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1445                 'duration': 220,
1446                 'upload_date': '20150625',
1447                 'uploader_id': 'dorappi2000',
1448                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1449                 'uploader': 'dorappi2000',
1450                 'formats': 'mincount:31',
1451             },
1452             'skip': 'not actual anymore',
1453         },
1454         # DASH manifest with segment_list
1455         {
1456             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1457             'md5': '8ce563a1d667b599d21064e982ab9e31',
1458             'info_dict': {
1459                 'id': 'CsmdDsKjzN8',
1460                 'ext': 'mp4',
1461                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1462                 'uploader': 'Airtek',
1463                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1464                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1465                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1466             },
1467             'params': {
1468                 'youtube_include_dash_manifest': True,
1469                 'format': '135',  # bestvideo
1470             },
1471             'skip': 'This live event has ended.',
1472         },
1473         {
1474             # Multifeed videos (multiple cameras), URL is for Main Camera
1475             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1476             'info_dict': {
1477                 'id': 'jvGDaLqkpTg',
1478                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1479                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1480             },
1481             'playlist': [{
1482                 'info_dict': {
1483                     'id': 'jvGDaLqkpTg',
1484                     'ext': 'mp4',
1485                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1486                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1487                     'duration': 10643,
1488                     'upload_date': '20161111',
1489                     'uploader': 'Team PGP',
1490                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1491                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1492                 },
1493             }, {
1494                 'info_dict': {
1495                     'id': '3AKt1R1aDnw',
1496                     'ext': 'mp4',
1497                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1498                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1499                     'duration': 10991,
1500                     'upload_date': '20161111',
1501                     'uploader': 'Team PGP',
1502                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1503                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1504                 },
1505             }, {
1506                 'info_dict': {
1507                     'id': 'RtAMM00gpVc',
1508                     'ext': 'mp4',
1509                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1510                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1511                     'duration': 10995,
1512                     'upload_date': '20161111',
1513                     'uploader': 'Team PGP',
1514                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1515                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1516                 },
1517             }, {
1518                 'info_dict': {
1519                     'id': '6N2fdlP3C5U',
1520                     'ext': 'mp4',
1521                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1522                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1523                     'duration': 10990,
1524                     'upload_date': '20161111',
1525                     'uploader': 'Team PGP',
1526                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1527                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1528                 },
1529             }],
1530             'params': {
1531                 'skip_download': True,
1532             },
1533             'skip': 'Not multifeed anymore',
1534         },
1535         {
1536             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1537             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1538             'info_dict': {
1539                 'id': 'gVfLd0zydlo',
1540                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1541             },
1542             'playlist_count': 2,
1543             'skip': 'Not multifeed anymore',
1544         },
1545         {
1546             'url': 'https://vid.plus/FlRa-iH7PGw',
1547             'only_matching': True,
1548         },
1549         {
1550             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1551             'only_matching': True,
1552         },
1553         {
1554             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1555             # Also tests cut-off URL expansion in video description (see
1556             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1557             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1558             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1559             'info_dict': {
1560                 'id': 'lsguqyKfVQg',
1561                 'ext': 'mp4',
1562                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1563                 'alt_title': 'Dark Walk',
1564                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1565                 'duration': 133,
1566                 'upload_date': '20151119',
1567                 'uploader_id': 'IronSoulElf',
1568                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1569                 'uploader': 'IronSoulElf',
1570                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1571                 'track': 'Dark Walk',
1572                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1573                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1574                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1575                 'categories': ['Film & Animation'],
1576                 'view_count': int,
1577                 'live_status': 'not_live',
1578                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1579                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1580                 'tags': 'count:13',
1581                 'availability': 'public',
1582                 'channel': 'IronSoulElf',
1583                 'playable_in_embed': True,
1584                 'like_count': int,
1585                 'age_limit': 0,
1586                 'channel_follower_count': int
1587             },
1588             'params': {
1589                 'skip_download': True,
1590             },
1591         },
1592         {
1593             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1594             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1595             'only_matching': True,
1596         },
1597         {
1598             # Video with yt:stretch=17:0
1599             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1600             'info_dict': {
1601                 'id': 'Q39EVAstoRM',
1602                 'ext': 'mp4',
1603                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1604                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1605                 'upload_date': '20151107',
1606                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1607                 'uploader': 'CH GAMER DROID',
1608             },
1609             'params': {
1610                 'skip_download': True,
1611             },
1612             'skip': 'This video does not exist.',
1613         },
1614         {
1615             # Video with incomplete 'yt:stretch=16:'
1616             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1617             'only_matching': True,
1618         },
1619         {
1620             # Video licensed under Creative Commons
1621             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1622             'info_dict': {
1623                 'id': 'M4gD1WSo5mA',
1624                 'ext': 'mp4',
1625                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1626                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1627                 'duration': 721,
1628                 'upload_date': '20150128',
1629                 'uploader_id': 'BerkmanCenter',
1630                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1631                 'uploader': 'The Berkman Klein Center for Internet & Society',
1632                 'license': 'Creative Commons Attribution license (reuse allowed)',
1633                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1634                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1635                 'like_count': int,
1636                 'age_limit': 0,
1637                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1638                 'channel': 'The Berkman Klein Center for Internet & Society',
1639                 'availability': 'public',
1640                 'view_count': int,
1641                 'categories': ['Education'],
1642                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1643                 'live_status': 'not_live',
1644                 'playable_in_embed': True,
1645                 'comment_count': int,
1646                 'channel_follower_count': int
1647             },
1648             'params': {
1649                 'skip_download': True,
1650             },
1651         },
1652         {
1653             # Channel-like uploader_url
1654             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1655             'info_dict': {
1656                 'id': 'eQcmzGIKrzg',
1657                 'ext': 'mp4',
1658                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1659                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1660                 'duration': 4060,
1661                 'upload_date': '20151120',
1662                 'uploader': 'Bernie Sanders',
1663                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1664                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1665                 'license': 'Creative Commons Attribution license (reuse allowed)',
1666                 'playable_in_embed': True,
1667                 'tags': 'count:12',
1668                 'like_count': int,
1669                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1670                 'age_limit': 0,
1671                 'availability': 'public',
1672                 'categories': ['News & Politics'],
1673                 'channel': 'Bernie Sanders',
1674                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1675                 'view_count': int,
1676                 'live_status': 'not_live',
1677                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1678                 'comment_count': int,
1679                 'channel_follower_count': int
1680             },
1681             'params': {
1682                 'skip_download': True,
1683             },
1684         },
1685         {
1686             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1687             'only_matching': True,
1688         },
1689         {
1690             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1691             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1692             'only_matching': True,
1693         },
1694         {
1695             # Rental video preview
1696             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1697             'info_dict': {
1698                 'id': 'uGpuVWrhIzE',
1699                 'ext': 'mp4',
1700                 'title': 'Piku - Trailer',
1701                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1702                 'upload_date': '20150811',
1703                 'uploader': 'FlixMatrix',
1704                 'uploader_id': 'FlixMatrixKaravan',
1705                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1706                 'license': 'Standard YouTube License',
1707             },
1708             'params': {
1709                 'skip_download': True,
1710             },
1711             'skip': 'This video is not available.',
1712         },
1713         {
1714             # YouTube Red video with episode data
1715             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1716             'info_dict': {
1717                 'id': 'iqKdEhx-dD4',
1718                 'ext': 'mp4',
1719                 'title': 'Isolation - Mind Field (Ep 1)',
1720                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1721                 'duration': 2085,
1722                 'upload_date': '20170118',
1723                 'uploader': 'Vsauce',
1724                 'uploader_id': 'Vsauce',
1725                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1726                 'series': 'Mind Field',
1727                 'season_number': 1,
1728                 'episode_number': 1,
1729                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1730                 'tags': 'count:12',
1731                 'view_count': int,
1732                 'availability': 'public',
1733                 'age_limit': 0,
1734                 'channel': 'Vsauce',
1735                 'episode': 'Episode 1',
1736                 'categories': ['Entertainment'],
1737                 'season': 'Season 1',
1738                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1739                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1740                 'like_count': int,
1741                 'playable_in_embed': True,
1742                 'live_status': 'not_live',
1743                 'channel_follower_count': int
1744             },
1745             'params': {
1746                 'skip_download': True,
1747             },
1748             'expected_warnings': [
1749                 'Skipping DASH manifest',
1750             ],
1751         },
1752         {
1753             # The following content has been identified by the YouTube community
1754             # as inappropriate or offensive to some audiences.
1755             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1756             'info_dict': {
1757                 'id': '6SJNVb0GnPI',
1758                 'ext': 'mp4',
1759                 'title': 'Race Differences in Intelligence',
1760                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1761                 'duration': 965,
1762                 'upload_date': '20140124',
1763                 'uploader': 'New Century Foundation',
1764                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1765                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1766             },
1767             'params': {
1768                 'skip_download': True,
1769             },
1770             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1771         },
1772         {
1773             # itag 212
1774             'url': '1t24XAntNCY',
1775             'only_matching': True,
1776         },
1777         {
1778             # geo restricted to JP
1779             'url': 'sJL6WA-aGkQ',
1780             'only_matching': True,
1781         },
1782         {
1783             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1784             'only_matching': True,
1785         },
1786         {
1787             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1788             'only_matching': True,
1789         },
1790         {
1791             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1792             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1793             'only_matching': True,
1794         },
1795         {
1796             # DRM protected
1797             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1798             'only_matching': True,
1799         },
1800         {
1801             # Video with unsupported adaptive stream type formats
1802             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1803             'info_dict': {
1804                 'id': 'Z4Vy8R84T1U',
1805                 'ext': 'mp4',
1806                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1807                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1808                 'duration': 433,
1809                 'upload_date': '20130923',
1810                 'uploader': 'Amelia Putri Harwita',
1811                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1812                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1813                 'formats': 'maxcount:10',
1814             },
1815             'params': {
1816                 'skip_download': True,
1817                 'youtube_include_dash_manifest': False,
1818             },
1819             'skip': 'not actual anymore',
1820         },
1821         {
1822             # Youtube Music Auto-generated description
1823             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1824             'info_dict': {
1825                 'id': 'MgNrAu2pzNs',
1826                 'ext': 'mp4',
1827                 'title': 'Voyeur Girl',
1828                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1829                 'upload_date': '20190312',
1830                 'uploader': 'Stephen - Topic',
1831                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1832                 'artist': 'Stephen',
1833                 'track': 'Voyeur Girl',
1834                 'album': 'it\'s too much love to know my dear',
1835                 'release_date': '20190313',
1836                 'release_year': 2019,
1837                 'alt_title': 'Voyeur Girl',
1838                 'view_count': int,
1839                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1840                 'playable_in_embed': True,
1841                 'like_count': int,
1842                 'categories': ['Music'],
1843                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1844                 'channel': 'Stephen',
1845                 'availability': 'public',
1846                 'creator': 'Stephen',
1847                 'duration': 169,
1848                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1849                 'age_limit': 0,
1850                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1851                 'tags': 'count:11',
1852                 'live_status': 'not_live',
1853                 'channel_follower_count': int
1854             },
1855             'params': {
1856                 'skip_download': True,
1857             },
1858         },
1859         {
1860             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1861             'only_matching': True,
1862         },
1863         {
1864             # invalid -> valid video id redirection
1865             'url': 'DJztXj2GPfl',
1866             'info_dict': {
1867                 'id': 'DJztXj2GPfk',
1868                 'ext': 'mp4',
1869                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1870                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1871                 'upload_date': '20090125',
1872                 'uploader': 'Prochorowka',
1873                 'uploader_id': 'Prochorowka',
1874                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1875                 'artist': 'Panjabi MC',
1876                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1877                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1878             },
1879             'params': {
1880                 'skip_download': True,
1881             },
1882             'skip': 'Video unavailable',
1883         },
1884         {
1885             # empty description results in an empty string
1886             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1887             'info_dict': {
1888                 'id': 'x41yOUIvK2k',
1889                 'ext': 'mp4',
1890                 'title': 'IMG 3456',
1891                 'description': '',
1892                 'upload_date': '20170613',
1893                 'uploader_id': 'ElevageOrVert',
1894                 'uploader': 'ElevageOrVert',
1895                 'view_count': int,
1896                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1897                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1898                 'like_count': int,
1899                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1900                 'tags': [],
1901                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1902                 'availability': 'public',
1903                 'age_limit': 0,
1904                 'categories': ['Pets & Animals'],
1905                 'duration': 7,
1906                 'playable_in_embed': True,
1907                 'live_status': 'not_live',
1908                 'channel': 'ElevageOrVert',
1909                 'channel_follower_count': int
1910             },
1911             'params': {
1912                 'skip_download': True,
1913             },
1914         },
1915         {
1916             # with '};' inside yt initial data (see [1])
1917             # see [2] for an example with '};' inside ytInitialPlayerResponse
1918             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1919             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1920             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1921             'info_dict': {
1922                 'id': 'CHqg6qOn4no',
1923                 'ext': 'mp4',
1924                 'title': 'Part 77   Sort a list of simple types in c#',
1925                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1926                 'upload_date': '20130831',
1927                 'uploader_id': 'kudvenkat',
1928                 'uploader': 'kudvenkat',
1929                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1930                 'like_count': int,
1931                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1932                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1933                 'live_status': 'not_live',
1934                 'categories': ['Education'],
1935                 'availability': 'public',
1936                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1937                 'tags': 'count:12',
1938                 'playable_in_embed': True,
1939                 'age_limit': 0,
1940                 'view_count': int,
1941                 'duration': 522,
1942                 'channel': 'kudvenkat',
1943                 'comment_count': int,
1944                 'channel_follower_count': int
1945             },
1946             'params': {
1947                 'skip_download': True,
1948             },
1949         },
1950         {
1951             # another example of '};' in ytInitialData
1952             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1953             'only_matching': True,
1954         },
1955         {
1956             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1957             'only_matching': True,
1958         },
1959         {
1960             # https://github.com/ytdl-org/youtube-dl/pull/28094
1961             'url': 'OtqTfy26tG0',
1962             'info_dict': {
1963                 'id': 'OtqTfy26tG0',
1964                 'ext': 'mp4',
1965                 'title': 'Burn Out',
1966                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1967                 'upload_date': '20141120',
1968                 'uploader': 'The Cinematic Orchestra - Topic',
1969                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1970                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1971                 'artist': 'The Cinematic Orchestra',
1972                 'track': 'Burn Out',
1973                 'album': 'Every Day',
1974                 'like_count': int,
1975                 'live_status': 'not_live',
1976                 'alt_title': 'Burn Out',
1977                 'duration': 614,
1978                 'age_limit': 0,
1979                 'view_count': int,
1980                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1981                 'creator': 'The Cinematic Orchestra',
1982                 'channel': 'The Cinematic Orchestra',
1983                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1984                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1985                 'availability': 'public',
1986                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1987                 'categories': ['Music'],
1988                 'playable_in_embed': True,
1989                 'channel_follower_count': int
1990             },
1991             'params': {
1992                 'skip_download': True,
1993             },
1994         },
1995         {
1996             # controversial video, only works with bpctr when authenticated with cookies
1997             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1998             'only_matching': True,
1999         },
2000         {
2001             # controversial video, requires bpctr/contentCheckOk
2002             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2003             'info_dict': {
2004                 'id': 'SZJvDhaSDnc',
2005                 'ext': 'mp4',
2006                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2007                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2008                 'uploader': 'CBS Mornings',
2009                 'uploader_id': 'CBSThisMorning',
2010                 'upload_date': '20140716',
2011                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2012                 'duration': 170,
2013                 'categories': ['News & Politics'],
2014                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2015                 'view_count': int,
2016                 'channel': 'CBS Mornings',
2017                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2018                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2019                 'age_limit': 18,
2020                 'availability': 'needs_auth',
2021                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2022                 'like_count': int,
2023                 'live_status': 'not_live',
2024                 'playable_in_embed': True,
2025                 'channel_follower_count': int
2026             }
2027         },
2028         {
2029             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2030             'url': 'cBvYw8_A0vQ',
2031             'info_dict': {
2032                 'id': 'cBvYw8_A0vQ',
2033                 'ext': 'mp4',
2034                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2035                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2036                 'upload_date': '20201120',
2037                 'uploader': 'Walk around Japan',
2038                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2039                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2040                 'duration': 1456,
2041                 'categories': ['Travel & Events'],
2042                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2043                 'view_count': int,
2044                 'channel': 'Walk around Japan',
2045                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2046                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2047                 'age_limit': 0,
2048                 'availability': 'public',
2049                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2050                 'live_status': 'not_live',
2051                 'playable_in_embed': True,
2052                 'channel_follower_count': int
2053             },
2054             'params': {
2055                 'skip_download': True,
2056             },
2057         }, {
2058             # Has multiple audio streams
2059             'url': 'WaOKSUlf4TM',
2060             'only_matching': True
2061         }, {
2062             # Requires Premium: has format 141 when requested using YTM url
2063             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2064             'only_matching': True
2065         }, {
2066             # multiple subtitles with same lang_code
2067             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2068             'only_matching': True,
2069         }, {
2070             # Force use android client fallback
2071             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2072             'info_dict': {
2073                 'id': 'YOelRv7fMxY',
2074                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2075                 'ext': '3gp',
2076                 'upload_date': '20210624',
2077                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2078                 'uploader': 'colinfurze',
2079                 'uploader_id': 'colinfurze',
2080                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2081                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2082                 'duration': 596,
2083                 'categories': ['Entertainment'],
2084                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2085                 'view_count': int,
2086                 'channel': 'colinfurze',
2087                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2088                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2089                 'age_limit': 0,
2090                 'availability': 'public',
2091                 'like_count': int,
2092                 'live_status': 'not_live',
2093                 'playable_in_embed': True,
2094                 'channel_follower_count': int
2095             },
2096             'params': {
2097                 'format': '17',  # 3gp format available on android
2098                 'extractor_args': {'youtube': {'player_client': ['android']}},
2099             },
2100         },
2101         {
2102             # Skip download of additional client configs (remix client config in this case)
2103             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2104             'only_matching': True,
2105             'params': {
2106                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2107             },
2108         }, {
2109             # shorts
2110             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2111             'only_matching': True,
2112         }, {
2113             'note': 'Storyboards',
2114             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2115             'info_dict': {
2116                 'id': '5KLPxDtMqe8',
2117                 'ext': 'mhtml',
2118                 'format_id': 'sb0',
2119                 'title': 'Your Brain is Plastic',
2120                 'uploader_id': 'scishow',
2121                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2122                 'upload_date': '20140324',
2123                 'uploader': 'SciShow',
2124                 'like_count': int,
2125                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2126                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2127                 'view_count': int,
2128                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2129                 'playable_in_embed': True,
2130                 'tags': 'count:12',
2131                 'uploader_url': 'http://www.youtube.com/user/scishow',
2132                 'availability': 'public',
2133                 'channel': 'SciShow',
2134                 'live_status': 'not_live',
2135                 'duration': 248,
2136                 'categories': ['Education'],
2137                 'age_limit': 0,
2138                 'channel_follower_count': int
2139             }, 'params': {'format': 'mhtml', 'skip_download': True}
2140         }, {
2141             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2142             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2143             'info_dict': {
2144                 'id': '2NUZ8W2llS4',
2145                 'ext': 'mp4',
2146                 'title': 'The NP that test your phone performance 🙂',
2147                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2148                 'uploader': 'Leon Nguyen',
2149                 'uploader_id': 'VNSXIII',
2150                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2151                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2152                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2153                 'duration': 21,
2154                 'view_count': int,
2155                 'age_limit': 0,
2156                 'categories': ['Gaming'],
2157                 'tags': 'count:23',
2158                 'playable_in_embed': True,
2159                 'live_status': 'not_live',
2160                 'upload_date': '20220103',
2161                 'like_count': int,
2162                 'availability': 'public',
2163                 'channel': 'Leon Nguyen',
2164                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2165                 'comment_count': int,
2166                 'channel_follower_count': int
2167             }
2168         }, {
2169             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2170             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2171             'info_dict': {
2172                 'id': 'mzZzzBU6lrM',
2173                 'ext': 'mp4',
2174                 'title': 'I Met GeorgeNotFound In Real Life...',
2175                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2176                 'uploader': 'Quackity',
2177                 'uploader_id': 'QuackityHQ',
2178                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2179                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2180                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2181                 'duration': 955,
2182                 'view_count': int,
2183                 'age_limit': 0,
2184                 'categories': ['Entertainment'],
2185                 'tags': 'count:26',
2186                 'playable_in_embed': True,
2187                 'live_status': 'not_live',
2188                 'release_timestamp': 1641172509,
2189                 'release_date': '20220103',
2190                 'upload_date': '20220103',
2191                 'like_count': int,
2192                 'availability': 'public',
2193                 'channel': 'Quackity',
2194                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2195                 'channel_follower_count': int
2196             }
2197         },
2198         {   # continuous livestream. Microformat upload date should be preferred.
2199             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2200             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2201             'info_dict': {
2202                 'id': 'kgx4WGK0oNU',
2203                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2204                 'ext': 'mp4',
2205                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2206                 'availability': 'public',
2207                 'age_limit': 0,
2208                 'release_timestamp': 1637975704,
2209                 'upload_date': '20210619',
2210                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2211                 'live_status': 'is_live',
2212                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2213                 'uploader': '阿鲍Abao',
2214                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2215                 'channel': 'Abao in Tokyo',
2216                 'channel_follower_count': int,
2217                 'release_date': '20211127',
2218                 'tags': 'count:39',
2219                 'categories': ['People & Blogs'],
2220                 'like_count': int,
2221                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2222                 'view_count': int,
2223                 'playable_in_embed': True,
2224                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2225             },
2226             'params': {'skip_download': True}
2227         }, {
2228             # Story. Requires specific player params to work.
2229             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2230             'info_dict': {
2231                 'id': 'vv8qTUWmulI',
2232                 'ext': 'mp4',
2233                 'availability': 'unlisted',
2234                 'view_count': int,
2235                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2236                 'upload_date': '20220526',
2237                 'categories': ['Education'],
2238                 'title': 'Story',
2239                 'channel': 'IT\'S HISTORY',
2240                 'description': '',
2241                 'uploader_id': 'BlastfromthePast',
2242                 'duration': 12,
2243                 'uploader': 'IT\'S HISTORY',
2244                 'playable_in_embed': True,
2245                 'age_limit': 0,
2246                 'live_status': 'not_live',
2247                 'tags': [],
2248                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2249                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2250                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2251             },
2252             'skip': 'stories get removed after some period of time',
2253         }, {
2254             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2255             'info_dict': {
2256                 'id': 'tjjjtzRLHvA',
2257                 'ext': 'mp4',
2258                 'title': 'ハッシュタグ無し };if window.ytcsi',
2259                 'upload_date': '20220323',
2260                 'like_count': int,
2261                 'availability': 'unlisted',
2262                 'channel': 'nao20010128nao',
2263                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2264                 'age_limit': 0,
2265                 'uploader': 'nao20010128nao',
2266                 'uploader_id': 'nao20010128nao',
2267                 'categories': ['Music'],
2268                 'view_count': int,
2269                 'description': '',
2270                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2271                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2272                 'live_status': 'not_live',
2273                 'playable_in_embed': True,
2274                 'channel_follower_count': int,
2275                 'duration': 6,
2276                 'tags': [],
2277                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2278             }
2279         }
2280     ]
2281
2282     _WEBPAGE_TESTS = [
2283         # YouTube <object> embed
2284         {
2285             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2286             'md5': '873c81d308b979f0e23ee7e620b312a3',
2287             'info_dict': {
2288                 'id': 'msN87y-iEx0',
2289                 'ext': 'mp4',
2290                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2291                 'upload_date': '20080526',
2292                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2293                 'uploader': 'Christopher Sykes',
2294                 'uploader_id': 'ChristopherJSykes',
2295                 'age_limit': 0,
2296                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2297                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2298                 'playable_in_embed': True,
2299                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2300                 'like_count': int,
2301                 'comment_count': int,
2302                 'channel': 'Christopher Sykes',
2303                 'live_status': 'not_live',
2304                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2305                 'availability': 'public',
2306                 'duration': 195,
2307                 'view_count': int,
2308                 'categories': ['Science & Technology'],
2309                 'channel_follower_count': int,
2310                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2311             },
2312             'params': {
2313                 'skip_download': True,
2314             }
2315         },
2316     ]
2317
2318     @classmethod
2319     def suitable(cls, url):
2320         from ..utils import parse_qs
2321
2322         qs = parse_qs(url)
2323         if qs.get('list', [None])[0]:
2324             return False
2325         return super().suitable(url)
2326
2327     def __init__(self, *args, **kwargs):
2328         super().__init__(*args, **kwargs)
2329         self._code_cache = {}
2330         self._player_cache = {}
2331
2332     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2333         lock = threading.Lock()
2334
2335         is_live = True
2336         start_time = time.time()
2337         formats = [f for f in formats if f.get('is_from_start')]
2338
2339         def refetch_manifest(format_id, delay):
2340             nonlocal formats, start_time, is_live
2341             if time.time() <= start_time + delay:
2342                 return
2343
2344             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2345             video_details = traverse_obj(
2346                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2347             microformats = traverse_obj(
2348                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2349                 expected_type=dict, default=[])
2350             _, is_live, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2351             start_time = time.time()
2352
2353         def mpd_feed(format_id, delay):
2354             """
2355             @returns (manifest_url, manifest_stream_number, is_live) or None
2356             """
2357             with lock:
2358                 refetch_manifest(format_id, delay)
2359
2360             f = next((f for f in formats if f['format_id'] == format_id), None)
2361             if not f:
2362                 if not is_live:
2363                     self.to_screen(f'{video_id}: Video is no longer live')
2364                 else:
2365                     self.report_warning(
2366                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2367                 return None
2368             return f['manifest_url'], f['manifest_stream_number'], is_live
2369
2370         for f in formats:
2371             f['is_live'] = True
2372             f['protocol'] = 'http_dash_segments_generator'
2373             f['fragments'] = functools.partial(
2374                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2375
2376     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2377         FETCH_SPAN, MAX_DURATION = 5, 432000
2378
2379         mpd_url, stream_number, is_live = None, None, True
2380
2381         begin_index = 0
2382         download_start_time = ctx.get('start') or time.time()
2383
2384         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2385         if lack_early_segments:
2386             self.report_warning(bug_reports_message(
2387                 'Starting download from the last 120 hours of the live stream since '
2388                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2389             lack_early_segments = True
2390
2391         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2392         fragments, fragment_base_url = None, None
2393
2394         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2395             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2396             # Obtain from MPD's maximum seq value
2397             old_mpd_url = mpd_url
2398             last_error = ctx.pop('last_error', None)
2399             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2400             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2401                                                or (mpd_url, stream_number, False))
2402             if not refresh_sequence:
2403                 if expire_fast and not is_live:
2404                     return False, last_seq
2405                 elif old_mpd_url == mpd_url:
2406                     return True, last_seq
2407             try:
2408                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2409                     mpd_url, None, note=False, errnote=False, fatal=False)
2410             except ExtractorError:
2411                 fmts = None
2412             if not fmts:
2413                 no_fragment_score += 2
2414                 return False, last_seq
2415             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2416             fragments = fmt_info['fragments']
2417             fragment_base_url = fmt_info['fragment_base_url']
2418             assert fragment_base_url
2419
2420             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2421             return True, _last_seq
2422
2423         while is_live:
2424             fetch_time = time.time()
2425             if no_fragment_score > 30:
2426                 return
2427             if last_segment_url:
2428                 # Obtain from "X-Head-Seqnum" header value from each segment
2429                 try:
2430                     urlh = self._request_webpage(
2431                         last_segment_url, None, note=False, errnote=False, fatal=False)
2432                 except ExtractorError:
2433                     urlh = None
2434                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2435                 if last_seq is None:
2436                     no_fragment_score += 2
2437                     last_segment_url = None
2438                     continue
2439             else:
2440                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2441                 no_fragment_score += 2
2442                 if not should_continue:
2443                     continue
2444
2445             if known_idx > last_seq:
2446                 last_segment_url = None
2447                 continue
2448
2449             last_seq += 1
2450
2451             if begin_index < 0 and known_idx < 0:
2452                 # skip from the start when it's negative value
2453                 known_idx = last_seq + begin_index
2454             if lack_early_segments:
2455                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2456             try:
2457                 for idx in range(known_idx, last_seq):
2458                     # do not update sequence here or you'll get skipped some part of it
2459                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2460                     if not should_continue:
2461                         known_idx = idx - 1
2462                         raise ExtractorError('breaking out of outer loop')
2463                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2464                     yield {
2465                         'url': last_segment_url,
2466                         'fragment_count': last_seq,
2467                     }
2468                 if known_idx == last_seq:
2469                     no_fragment_score += 5
2470                 else:
2471                     no_fragment_score = 0
2472                 known_idx = last_seq
2473             except ExtractorError:
2474                 continue
2475
2476             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2477
2478     def _extract_player_url(self, *ytcfgs, webpage=None):
2479         player_url = traverse_obj(
2480             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2481             get_all=False, expected_type=str)
2482         if not player_url:
2483             return
2484         return urljoin('https://www.youtube.com', player_url)
2485
2486     def _download_player_url(self, video_id, fatal=False):
2487         res = self._download_webpage(
2488             'https://www.youtube.com/iframe_api',
2489             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2490         if res:
2491             player_version = self._search_regex(
2492                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2493             if player_version:
2494                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2495
2496     def _signature_cache_id(self, example_sig):
2497         """ Return a string representation of a signature """
2498         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2499
2500     @classmethod
2501     def _extract_player_info(cls, player_url):
2502         for player_re in cls._PLAYER_INFO_RE:
2503             id_m = re.search(player_re, player_url)
2504             if id_m:
2505                 break
2506         else:
2507             raise ExtractorError('Cannot identify player %r' % player_url)
2508         return id_m.group('id')
2509
2510     def _load_player(self, video_id, player_url, fatal=True):
2511         player_id = self._extract_player_info(player_url)
2512         if player_id not in self._code_cache:
2513             code = self._download_webpage(
2514                 player_url, video_id, fatal=fatal,
2515                 note='Downloading player ' + player_id,
2516                 errnote='Download of %s failed' % player_url)
2517             if code:
2518                 self._code_cache[player_id] = code
2519         return self._code_cache.get(player_id)
2520
2521     def _extract_signature_function(self, video_id, player_url, example_sig):
2522         player_id = self._extract_player_info(player_url)
2523
2524         # Read from filesystem cache
2525         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2526         assert os.path.basename(func_id) == func_id
2527
2528         self.write_debug(f'Extracting signature function {func_id}')
2529         cache_spec = self.cache.load('youtube-sigfuncs', func_id)
2530         if cache_spec is not None:
2531             return lambda s: ''.join(s[i] for i in cache_spec)
2532
2533         code = self._load_player(video_id, player_url)
2534         if code:
2535             res = self._parse_sig_js(code)
2536
2537             test_string = ''.join(map(chr, range(len(example_sig))))
2538             cache_res = res(test_string)
2539             cache_spec = [ord(c) for c in cache_res]
2540
2541             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2542             return res
2543
2544     def _print_sig_code(self, func, example_sig):
2545         if not self.get_param('youtube_print_sig_code'):
2546             return
2547
2548         def gen_sig_code(idxs):
2549             def _genslice(start, end, step):
2550                 starts = '' if start == 0 else str(start)
2551                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2552                 steps = '' if step == 1 else (':%d' % step)
2553                 return f's[{starts}{ends}{steps}]'
2554
2555             step = None
2556             # Quelch pyflakes warnings - start will be set when step is set
2557             start = '(Never used)'
2558             for i, prev in zip(idxs[1:], idxs[:-1]):
2559                 if step is not None:
2560                     if i - prev == step:
2561                         continue
2562                     yield _genslice(start, prev, step)
2563                     step = None
2564                     continue
2565                 if i - prev in [-1, 1]:
2566                     step = i - prev
2567                     start = prev
2568                     continue
2569                 else:
2570                     yield 's[%d]' % prev
2571             if step is None:
2572                 yield 's[%d]' % i
2573             else:
2574                 yield _genslice(start, i, step)
2575
2576         test_string = ''.join(map(chr, range(len(example_sig))))
2577         cache_res = func(test_string)
2578         cache_spec = [ord(c) for c in cache_res]
2579         expr_code = ' + '.join(gen_sig_code(cache_spec))
2580         signature_id_tuple = '(%s)' % (
2581             ', '.join(str(len(p)) for p in example_sig.split('.')))
2582         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2583                 '    return %s\n') % (signature_id_tuple, expr_code)
2584         self.to_screen('Extracted signature function:\n' + code)
2585
2586     def _parse_sig_js(self, jscode):
2587         funcname = self._search_regex(
2588             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2589              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2590              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2591              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2592              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2593              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2594              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2595              # Obsolete patterns
2596              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2597              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2598              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2599              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2600              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2601              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2602              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2603              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2604             jscode, 'Initial JS player signature function name', group='sig')
2605
2606         jsi = JSInterpreter(jscode)
2607         initial_function = jsi.extract_function(funcname)
2608         return lambda s: initial_function([s])
2609
2610     def _decrypt_signature(self, s, video_id, player_url):
2611         """Turn the encrypted s field into a working signature"""
2612         try:
2613             player_id = (player_url, self._signature_cache_id(s))
2614             if player_id not in self._player_cache:
2615                 func = self._extract_signature_function(video_id, player_url, s)
2616                 self._player_cache[player_id] = func
2617             func = self._player_cache[player_id]
2618             self._print_sig_code(func, s)
2619             return func(s)
2620         except Exception as e:
2621             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2622
2623     def _decrypt_nsig(self, s, video_id, player_url):
2624         """Turn the encrypted n field into a working signature"""
2625         if player_url is None:
2626             raise ExtractorError('Cannot decrypt nsig without player_url')
2627         player_url = urljoin('https://www.youtube.com', player_url)
2628
2629         sig_id = ('nsig_value', s)
2630         if sig_id in self._player_cache:
2631             return self._player_cache[sig_id]
2632
2633         try:
2634             player_id = ('nsig', player_url)
2635             if player_id not in self._player_cache:
2636                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2637             func = self._player_cache[player_id]
2638             self._player_cache[sig_id] = func(s)
2639             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2640             return self._player_cache[sig_id]
2641         except Exception as e:
2642             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2643
2644     def _extract_n_function_name(self, jscode):
2645         nfunc, idx = self._search_regex(
2646             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2647             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2648         if not idx:
2649             return nfunc
2650         return json.loads(js_to_json(self._search_regex(
2651             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2652             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2653
2654     def _extract_n_function(self, video_id, player_url):
2655         player_id = self._extract_player_info(player_url)
2656         func_code = self.cache.load('youtube-nsig', player_id)
2657
2658         if func_code:
2659             jsi = JSInterpreter(func_code)
2660         else:
2661             jscode = self._load_player(video_id, player_url)
2662             funcname = self._extract_n_function_name(jscode)
2663             jsi = JSInterpreter(jscode)
2664             func_code = jsi.extract_function_code(funcname)
2665             self.cache.store('youtube-nsig', player_id, func_code)
2666
2667         if self.get_param('youtube_print_sig_code'):
2668             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2669
2670         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2671
2672     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2673         """
2674         Extract signatureTimestamp (sts)
2675         Required to tell API what sig/player version is in use.
2676         """
2677         sts = None
2678         if isinstance(ytcfg, dict):
2679             sts = int_or_none(ytcfg.get('STS'))
2680
2681         if not sts:
2682             # Attempt to extract from player
2683             if player_url is None:
2684                 error_msg = 'Cannot extract signature timestamp without player_url.'
2685                 if fatal:
2686                     raise ExtractorError(error_msg)
2687                 self.report_warning(error_msg)
2688                 return
2689             code = self._load_player(video_id, player_url, fatal=fatal)
2690             if code:
2691                 sts = int_or_none(self._search_regex(
2692                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2693                     'JS player signature timestamp', group='sts', fatal=fatal))
2694         return sts
2695
2696     def _mark_watched(self, video_id, player_responses):
2697         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2698             label = 'fully ' if is_full else ''
2699             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2700                             expected_type=url_or_none)
2701             if not url:
2702                 self.report_warning(f'Unable to mark {label}watched')
2703                 return
2704             parsed_url = urllib.parse.urlparse(url)
2705             qs = urllib.parse.parse_qs(parsed_url.query)
2706
2707             # cpn generation algorithm is reverse engineered from base.js.
2708             # In fact it works even with dummy cpn.
2709             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2710             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2711
2712             # # more consistent results setting it to right before the end
2713             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2714
2715             qs.update({
2716                 'ver': ['2'],
2717                 'cpn': [cpn],
2718                 'cmt': video_length,
2719                 'el': 'detailpage',  # otherwise defaults to "shorts"
2720             })
2721
2722             if is_full:
2723                 # these seem to mark watchtime "history" in the real world
2724                 # they're required, so send in a single value
2725                 qs.update({
2726                     'st': video_length,
2727                     'et': video_length,
2728                 })
2729
2730             url = urllib.parse.urlunparse(
2731                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2732
2733             self._download_webpage(
2734                 url, video_id, f'Marking {label}watched',
2735                 'Unable to mark watched', fatal=False)
2736
2737     @classmethod
2738     def _extract_from_webpage(cls, url, webpage):
2739         # Invidious Instances
2740         # https://github.com/yt-dlp/yt-dlp/issues/195
2741         # https://github.com/iv-org/invidious/pull/1730
2742         mobj = re.search(
2743             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
2744             webpage)
2745         if mobj:
2746             yield cls.url_result(mobj.group('url'), cls)
2747             raise cls.StopExtraction()
2748
2749         yield from super()._extract_from_webpage(url, webpage)
2750
2751         # lazyYT YouTube embed
2752         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
2753             yield cls.url_result(unescapeHTML(id_), cls, id_)
2754
2755         # Wordpress "YouTube Video Importer" plugin
2756         for m in re.findall(r'''(?x)<div[^>]+
2757                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2758                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
2759             yield cls.url_result(m[-1], cls, m[-1])
2760
2761     @classmethod
2762     def extract_id(cls, url):
2763         video_id = cls.get_temp_id(url)
2764         if not video_id:
2765             raise ExtractorError(f'Invalid URL: {url}')
2766         return video_id
2767
2768     def _extract_chapters_from_json(self, data, duration):
2769         chapter_list = traverse_obj(
2770             data, (
2771                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2772                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2773             ), expected_type=list)
2774
2775         return self._extract_chapters(
2776             chapter_list,
2777             chapter_time=lambda chapter: float_or_none(
2778                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2779             chapter_title=lambda chapter: traverse_obj(
2780                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2781             duration=duration)
2782
2783     def _extract_chapters_from_engagement_panel(self, data, duration):
2784         content_list = traverse_obj(
2785             data,
2786             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2787             expected_type=list, default=[])
2788         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2789         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2790
2791         return next(filter(None, (
2792             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2793                                    chapter_time, chapter_title, duration)
2794             for contents in content_list)), [])
2795
2796     def _extract_chapters_from_description(self, description, duration):
2797         return self._extract_chapters(
2798             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
2799             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
2800             duration=duration, strict=False)
2801
2802     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
2803         if not duration:
2804             return
2805         chapter_list = [{
2806             'start_time': chapter_time(chapter),
2807             'title': chapter_title(chapter),
2808         } for chapter in chapter_list or []]
2809         if not strict:
2810             chapter_list.sort(key=lambda c: c['start_time'] or 0)
2811
2812         chapters = [{'start_time': 0}]
2813         for idx, chapter in enumerate(chapter_list):
2814             if chapter['start_time'] is None:
2815                 self.report_warning(f'Incomplete chapter {idx}')
2816             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
2817                 chapters.append(chapter)
2818             else:
2819                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
2820         return chapters[1:]
2821
2822     def _extract_comment(self, comment_renderer, parent=None):
2823         comment_id = comment_renderer.get('commentId')
2824         if not comment_id:
2825             return
2826
2827         text = self._get_text(comment_renderer, 'contentText')
2828
2829         # note: timestamp is an estimate calculated from the current time and time_text
2830         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2831         author = self._get_text(comment_renderer, 'authorText')
2832         author_id = try_get(comment_renderer,
2833                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
2834
2835         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2836                                                        lambda x: x['likeCount']), str)) or 0
2837         author_thumbnail = try_get(comment_renderer,
2838                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
2839
2840         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2841         is_favorited = 'creatorHeart' in (try_get(
2842             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2843         return {
2844             'id': comment_id,
2845             'text': text,
2846             'timestamp': timestamp,
2847             'time_text': time_text,
2848             'like_count': votes,
2849             'is_favorited': is_favorited,
2850             'author': author,
2851             'author_id': author_id,
2852             'author_thumbnail': author_thumbnail,
2853             'author_is_uploader': author_is_uploader,
2854             'parent': parent or 'root'
2855         }
2856
2857     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2858
2859         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2860
2861         def extract_header(contents):
2862             _continuation = None
2863             for content in contents:
2864                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2865                 expected_comment_count = self._get_count(
2866                     comments_header_renderer, 'countText', 'commentsCount')
2867
2868                 if expected_comment_count:
2869                     tracker['est_total'] = expected_comment_count
2870                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2871                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2872
2873                 sort_menu_item = try_get(
2874                     comments_header_renderer,
2875                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2876                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2877
2878                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2879                 if not _continuation:
2880                     continue
2881
2882                 sort_text = str_or_none(sort_menu_item.get('title'))
2883                 if not sort_text:
2884                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2885                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2886                 break
2887             return _continuation
2888
2889         def extract_thread(contents):
2890             if not parent:
2891                 tracker['current_page_thread'] = 0
2892             for content in contents:
2893                 if not parent and tracker['total_parent_comments'] >= max_parents:
2894                     yield
2895                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2896                 comment_renderer = get_first(
2897                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2898                     expected_type=dict, default={})
2899
2900                 comment = self._extract_comment(comment_renderer, parent)
2901                 if not comment:
2902                     continue
2903
2904                 tracker['running_total'] += 1
2905                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2906                 yield comment
2907
2908                 # Attempt to get the replies
2909                 comment_replies_renderer = try_get(
2910                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2911
2912                 if comment_replies_renderer:
2913                     tracker['current_page_thread'] += 1
2914                     comment_entries_iter = self._comment_entries(
2915                         comment_replies_renderer, ytcfg, video_id,
2916                         parent=comment.get('id'), tracker=tracker)
2917                     yield from itertools.islice(comment_entries_iter, min(
2918                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2919
2920         # Keeps track of counts across recursive calls
2921         if not tracker:
2922             tracker = dict(
2923                 running_total=0,
2924                 est_total=0,
2925                 current_page_thread=0,
2926                 total_parent_comments=0,
2927                 total_reply_comments=0)
2928
2929         # TODO: Deprecated
2930         # YouTube comments have a max depth of 2
2931         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2932         if max_depth:
2933             self._downloader.deprecation_warning(
2934                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2935         if max_depth == 1 and parent:
2936             return
2937
2938         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2939             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2940
2941         continuation = self._extract_continuation(root_continuation_data)
2942
2943         response = None
2944         is_forced_continuation = False
2945         is_first_continuation = parent is None
2946         if is_first_continuation and not continuation:
2947             # Sometimes you can get comments by generating the continuation yourself,
2948             # even if YouTube initially reports them being disabled - e.g. stories comments.
2949             # Note: if the comment section is actually disabled, YouTube may return a response with
2950             # required check_get_keys missing. So we will disable that check initially in this case.
2951             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
2952             is_forced_continuation = True
2953
2954         for page_num in itertools.count(0):
2955             if not continuation:
2956                 break
2957             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2958             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2959             if page_num == 0:
2960                 if is_first_continuation:
2961                     note_prefix = 'Downloading comment section API JSON'
2962                 else:
2963                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2964                         tracker['current_page_thread'], comment_prog_str)
2965             else:
2966                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2967                     '       ' if parent else '', ' replies' if parent else '',
2968                     page_num, comment_prog_str)
2969
2970             response = self._extract_response(
2971                 item_id=None, query=continuation,
2972                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2973                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
2974             is_forced_continuation = False
2975             continuation_contents = traverse_obj(
2976                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2977
2978             continuation = None
2979             for continuation_section in continuation_contents:
2980                 continuation_items = traverse_obj(
2981                     continuation_section,
2982                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2983                     get_all=False, expected_type=list) or []
2984                 if is_first_continuation:
2985                     continuation = extract_header(continuation_items)
2986                     is_first_continuation = False
2987                     if continuation:
2988                         break
2989                     continue
2990
2991                 for entry in extract_thread(continuation_items):
2992                     if not entry:
2993                         return
2994                     yield entry
2995                 continuation = self._extract_continuation({'contents': continuation_items})
2996                 if continuation:
2997                     break
2998
2999         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3000         if message and not parent and tracker['running_total'] == 0:
3001             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3002
3003     @staticmethod
3004     def _generate_comment_continuation(video_id):
3005         """
3006         Generates initial comment section continuation token from given video id
3007         """
3008         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3009         return base64.b64encode(token.encode()).decode()
3010
3011     def _get_comments(self, ytcfg, video_id, contents, webpage):
3012         """Entry for comment extraction"""
3013         def _real_comment_extract(contents):
3014             renderer = next((
3015                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3016                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3017             yield from self._comment_entries(renderer, ytcfg, video_id)
3018
3019         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3020         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3021
3022     @staticmethod
3023     def _get_checkok_params():
3024         return {'contentCheckOk': True, 'racyCheckOk': True}
3025
3026     @classmethod
3027     def _generate_player_context(cls, sts=None):
3028         context = {
3029             'html5Preference': 'HTML5_PREF_WANTS',
3030         }
3031         if sts is not None:
3032             context['signatureTimestamp'] = sts
3033         return {
3034             'playbackContext': {
3035                 'contentPlaybackContext': context
3036             },
3037             **cls._get_checkok_params()
3038         }
3039
3040     @staticmethod
3041     def _is_agegated(player_response):
3042         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3043             return True
3044
3045         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3046         AGE_GATE_REASONS = (
3047             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3048             'age_verification_required', 'age_check_required',  # status
3049         )
3050         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3051
3052     @staticmethod
3053     def _is_unplayable(player_response):
3054         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3055
3056     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
3057
3058         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3059         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3060         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3061         headers = self.generate_api_headers(
3062             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3063
3064         yt_query = {
3065             'videoId': video_id,
3066             'params': '8AEB'  # enable stories
3067         }
3068         yt_query.update(self._generate_player_context(sts))
3069         return self._extract_response(
3070             item_id=video_id, ep='player', query=yt_query,
3071             ytcfg=player_ytcfg, headers=headers, fatal=True,
3072             default_client=client,
3073             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3074         ) or None
3075
3076     def _get_requested_clients(self, url, smuggled_data):
3077         requested_clients = []
3078         default = ['android', 'web']
3079         allowed_clients = sorted(
3080             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3081             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3082         for client in self._configuration_arg('player_client'):
3083             if client in allowed_clients:
3084                 requested_clients.append(client)
3085             elif client == 'default':
3086                 requested_clients.extend(default)
3087             elif client == 'all':
3088                 requested_clients.extend(allowed_clients)
3089             else:
3090                 self.report_warning(f'Skipping unsupported client {client}')
3091         if not requested_clients:
3092             requested_clients = default
3093
3094         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3095             requested_clients.extend(
3096                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3097
3098         return orderedSet(requested_clients)
3099
3100     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
3101         initial_pr = None
3102         if webpage:
3103             initial_pr = self._search_json(
3104                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3105
3106         all_clients = set(clients)
3107         clients = clients[::-1]
3108         prs = []
3109
3110         def append_client(*client_names):
3111             """ Append the first client name that exists but not already used """
3112             for client_name in client_names:
3113                 actual_client = _split_innertube_client(client_name)[0]
3114                 if actual_client in INNERTUBE_CLIENTS:
3115                     if actual_client not in all_clients:
3116                         clients.append(client_name)
3117                         all_clients.add(actual_client)
3118                         return
3119
3120         # Android player_response does not have microFormats which are needed for
3121         # extraction of some data. So we return the initial_pr with formats
3122         # stripped out even if not requested by the user
3123         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3124         if initial_pr:
3125             pr = dict(initial_pr)
3126             pr['streamingData'] = None
3127             prs.append(pr)
3128
3129         last_error = None
3130         tried_iframe_fallback = False
3131         player_url = None
3132         while clients:
3133             client, base_client, variant = _split_innertube_client(clients.pop())
3134             player_ytcfg = master_ytcfg if client == 'web' else {}
3135             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3136                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3137
3138             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3139             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3140             if 'js' in self._configuration_arg('player_skip'):
3141                 require_js_player = False
3142                 player_url = None
3143
3144             if not player_url and not tried_iframe_fallback and require_js_player:
3145                 player_url = self._download_player_url(video_id)
3146                 tried_iframe_fallback = True
3147
3148             try:
3149                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3150                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3151             except ExtractorError as e:
3152                 if last_error:
3153                     self.report_warning(last_error)
3154                 last_error = e
3155                 continue
3156
3157             if pr:
3158                 prs.append(pr)
3159
3160             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3161             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3162                 append_client(f'{base_client}_creator')
3163             elif self._is_agegated(pr):
3164                 if variant == 'tv_embedded':
3165                     append_client(f'{base_client}_embedded')
3166                 elif not variant:
3167                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3168
3169         if last_error:
3170             if not len(prs):
3171                 raise last_error
3172             self.report_warning(last_error)
3173         return prs, player_url
3174
3175     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, is_live, duration):
3176         itags, stream_ids = {}, []
3177         itag_qualities, res_qualities = {}, {}
3178         q = qualities([
3179             # Normally tiny is the smallest video-only formats. But
3180             # audio-only formats with unknown quality may get tagged as tiny
3181             'tiny',
3182             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3183             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3184         ])
3185         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3186
3187         for fmt in streaming_formats:
3188             if fmt.get('targetDurationSec'):
3189                 continue
3190
3191             itag = str_or_none(fmt.get('itag'))
3192             audio_track = fmt.get('audioTrack') or {}
3193             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3194             if stream_id in stream_ids:
3195                 continue
3196
3197             quality = fmt.get('quality')
3198             height = int_or_none(fmt.get('height'))
3199             if quality == 'tiny' or not quality:
3200                 quality = fmt.get('audioQuality', '').lower() or quality
3201             # The 3gp format (17) in android client has a quality of "small",
3202             # but is actually worse than other formats
3203             if itag == '17':
3204                 quality = 'tiny'
3205             if quality:
3206                 if itag:
3207                     itag_qualities[itag] = quality
3208                 if height:
3209                     res_qualities[height] = quality
3210             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3211             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3212             # number of fragment that would subsequently requested with (`&sq=N`)
3213             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3214                 continue
3215
3216             fmt_url = fmt.get('url')
3217             if not fmt_url:
3218                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3219                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3220                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3221                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3222                     continue
3223                 try:
3224                     fmt_url += '&%s=%s' % (
3225                         traverse_obj(sc, ('sp', -1)) or 'signature',
3226                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3227                     )
3228                 except ExtractorError as e:
3229                     self.report_warning('Signature extraction failed: Some formats may be missing', only_once=True)
3230                     self.write_debug(e, only_once=True)
3231                     continue
3232
3233             query = parse_qs(fmt_url)
3234             throttled = False
3235             if query.get('n'):
3236                 try:
3237                     fmt_url = update_url_query(fmt_url, {
3238                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3239                 except ExtractorError as e:
3240                     self.report_warning(
3241                         'nsig extraction failed: You may experience throttling for some formats\n'
3242                         f'n = {query["n"][0]} ; player = {player_url}', only_once=True)
3243                     self.write_debug(e, only_once=True)
3244                     throttled = True
3245
3246             if itag:
3247                 itags[itag] = 'https'
3248                 stream_ids.append(stream_id)
3249
3250             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3251             language_preference = (
3252                 10 if audio_track.get('audioIsDefault') and 10
3253                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3254                 else -1)
3255             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3256             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3257             # Make sure to avoid false positives with small duration differences.
3258             # Eg: __2ABJjxzNo, ySuUZEjARPY
3259             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3260             if is_damaged:
3261                 self.report_warning(
3262                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3263             dct = {
3264                 'asr': int_or_none(fmt.get('audioSampleRate')),
3265                 'filesize': int_or_none(fmt.get('contentLength')),
3266                 'format_id': itag,
3267                 'format_note': join_nonempty(
3268                     '%s%s' % (audio_track.get('displayName') or '',
3269                               ' (default)' if language_preference > 0 else ''),
3270                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3271                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3272                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3273                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3274                 'fps': int_or_none(fmt.get('fps')) or None,
3275                 'height': height,
3276                 'quality': q(quality),
3277                 'has_drm': bool(fmt.get('drmFamilies')),
3278                 'tbr': tbr,
3279                 'url': fmt_url,
3280                 'width': int_or_none(fmt.get('width')),
3281                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3282                                           'desc' if language_preference < -1 else ''),
3283                 'language_preference': language_preference,
3284                 # Strictly de-prioritize damaged and 3gp formats
3285                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3286             }
3287             mime_mobj = re.match(
3288                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3289             if mime_mobj:
3290                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3291                 dct.update(parse_codecs(mime_mobj.group(2)))
3292             no_audio = dct.get('acodec') == 'none'
3293             no_video = dct.get('vcodec') == 'none'
3294             if no_audio:
3295                 dct['vbr'] = tbr
3296             if no_video:
3297                 dct['abr'] = tbr
3298             if no_audio or no_video:
3299                 dct['downloader_options'] = {
3300                     # Youtube throttles chunks >~10M
3301                     'http_chunk_size': 10485760,
3302                 }
3303                 if dct.get('ext'):
3304                     dct['container'] = dct['ext'] + '_dash'
3305             yield dct
3306
3307         live_from_start = is_live and self.get_param('live_from_start')
3308         skip_manifests = self._configuration_arg('skip')
3309         if not self.get_param('youtube_include_hls_manifest', True):
3310             skip_manifests.append('hls')
3311         if not self.get_param('youtube_include_dash_manifest', True):
3312             skip_manifests.append('dash')
3313         get_dash = 'dash' not in skip_manifests and (
3314             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3315         get_hls = not live_from_start and 'hls' not in skip_manifests
3316
3317         def process_manifest_format(f, proto, itag):
3318             if itag in itags:
3319                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3320                     return False
3321                 itag = f'{itag}-{proto}'
3322             if itag:
3323                 f['format_id'] = itag
3324                 itags[itag] = proto
3325
3326             f['quality'] = next((
3327                 q(qdict[val])
3328                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3329                 if val in qdict), -1)
3330             return True
3331
3332         subtitles = {}
3333         for sd in streaming_data:
3334             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3335             if hls_manifest_url:
3336                 fmts, subs = self._extract_m3u8_formats_and_subtitles(hls_manifest_url, video_id, 'mp4', fatal=False, live=is_live)
3337                 subtitles = self._merge_subtitles(subs, subtitles)
3338                 for f in fmts:
3339                     if process_manifest_format(f, 'hls', self._search_regex(
3340                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3341                         yield f
3342
3343             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3344             if dash_manifest_url:
3345                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3346                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3347                 for f in formats:
3348                     if process_manifest_format(f, 'dash', f['format_id']):
3349                         f['filesize'] = int_or_none(self._search_regex(
3350                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3351                         if live_from_start:
3352                             f['is_from_start'] = True
3353
3354                         yield f
3355         yield subtitles
3356
3357     def _extract_storyboard(self, player_responses, duration):
3358         spec = get_first(
3359             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3360         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3361         if not base_url:
3362             return
3363         L = len(spec) - 1
3364         for i, args in enumerate(spec):
3365             args = args.split('#')
3366             counts = list(map(int_or_none, args[:5]))
3367             if len(args) != 8 or not all(counts):
3368                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3369                 continue
3370             width, height, frame_count, cols, rows = counts
3371             N, sigh = args[6:]
3372
3373             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3374             fragment_count = frame_count / (cols * rows)
3375             fragment_duration = duration / fragment_count
3376             yield {
3377                 'format_id': f'sb{i}',
3378                 'format_note': 'storyboard',
3379                 'ext': 'mhtml',
3380                 'protocol': 'mhtml',
3381                 'acodec': 'none',
3382                 'vcodec': 'none',
3383                 'url': url,
3384                 'width': width,
3385                 'height': height,
3386                 'fps': frame_count / duration,
3387                 'rows': rows,
3388                 'columns': cols,
3389                 'fragments': [{
3390                     'url': url.replace('$M', str(j)),
3391                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3392                 } for j in range(math.ceil(fragment_count))],
3393             }
3394
3395     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3396         webpage = None
3397         if 'webpage' not in self._configuration_arg('player_skip'):
3398             webpage = self._download_webpage(
3399                 webpage_url + '&bpctr=9999999999&has_verified=1&pp=8AEB', video_id, fatal=False)
3400
3401         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3402
3403         player_responses, player_url = self._extract_player_responses(
3404             self._get_requested_clients(url, smuggled_data),
3405             video_id, webpage, master_ytcfg)
3406
3407         return webpage, master_ytcfg, player_responses, player_url
3408
3409     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3410         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3411         is_live = get_first(video_details, 'isLive')
3412         if is_live is None:
3413             is_live = get_first(live_broadcast_details, 'isLiveNow')
3414
3415         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3416         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, is_live, duration)
3417
3418         return live_broadcast_details, is_live, streaming_data, formats, subtitles
3419
3420     def _real_extract(self, url):
3421         url, smuggled_data = unsmuggle_url(url, {})
3422         video_id = self._match_id(url)
3423
3424         base_url = self.http_scheme() + '//www.youtube.com/'
3425         webpage_url = base_url + 'watch?v=' + video_id
3426
3427         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3428
3429         playability_statuses = traverse_obj(
3430             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3431
3432         trailer_video_id = get_first(
3433             playability_statuses,
3434             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3435             expected_type=str)
3436         if trailer_video_id:
3437             return self.url_result(
3438                 trailer_video_id, self.ie_key(), trailer_video_id)
3439
3440         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3441                        if webpage else (lambda x: None))
3442
3443         video_details = traverse_obj(
3444             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3445         microformats = traverse_obj(
3446             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3447             expected_type=dict, default=[])
3448         video_title = (
3449             get_first(video_details, 'title')
3450             or self._get_text(microformats, (..., 'title'))
3451             or search_meta(['og:title', 'twitter:title', 'title']))
3452         video_description = get_first(video_details, 'shortDescription')
3453
3454         multifeed_metadata_list = get_first(
3455             player_responses,
3456             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3457             expected_type=str)
3458         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3459             if self.get_param('noplaylist'):
3460                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3461             else:
3462                 entries = []
3463                 feed_ids = []
3464                 for feed in multifeed_metadata_list.split(','):
3465                     # Unquote should take place before split on comma (,) since textual
3466                     # fields may contain comma as well (see
3467                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3468                     feed_data = urllib.parse.parse_qs(
3469                         urllib.parse.unquote_plus(feed))
3470
3471                     def feed_entry(name):
3472                         return try_get(
3473                             feed_data, lambda x: x[name][0], str)
3474
3475                     feed_id = feed_entry('id')
3476                     if not feed_id:
3477                         continue
3478                     feed_title = feed_entry('title')
3479                     title = video_title
3480                     if feed_title:
3481                         title += ' (%s)' % feed_title
3482                     entries.append({
3483                         '_type': 'url_transparent',
3484                         'ie_key': 'Youtube',
3485                         'url': smuggle_url(
3486                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3487                             {'force_singlefeed': True}),
3488                         'title': title,
3489                     })
3490                     feed_ids.append(feed_id)
3491                 self.to_screen(
3492                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3493                     % (', '.join(feed_ids), video_id))
3494                 return self.playlist_result(
3495                     entries, video_id, video_title, video_description)
3496
3497         duration = int_or_none(
3498             get_first(video_details, 'lengthSeconds')
3499             or get_first(microformats, 'lengthSeconds')
3500             or parse_duration(search_meta('duration'))) or None
3501
3502         live_broadcast_details, is_live, streaming_data, formats, automatic_captions = \
3503             self._list_formats(video_id, microformats, video_details, player_responses, player_url)
3504
3505         if not formats:
3506             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3507                 self.report_drm(video_id)
3508             pemr = get_first(
3509                 playability_statuses,
3510                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3511             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3512             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3513             if subreason:
3514                 if subreason == 'The uploader has not made this video available in your country.':
3515                     countries = get_first(microformats, 'availableCountries')
3516                     if not countries:
3517                         regions_allowed = search_meta('regionsAllowed')
3518                         countries = regions_allowed.split(',') if regions_allowed else None
3519                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3520                 reason += f'. {subreason}'
3521             if reason:
3522                 self.raise_no_formats(reason, expected=True)
3523
3524         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3525         if not keywords and webpage:
3526             keywords = [
3527                 unescapeHTML(m.group('content'))
3528                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3529         for keyword in keywords:
3530             if keyword.startswith('yt:stretch='):
3531                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3532                 if mobj:
3533                     # NB: float is intentional for forcing float division
3534                     w, h = (float(v) for v in mobj.groups())
3535                     if w > 0 and h > 0:
3536                         ratio = w / h
3537                         for f in formats:
3538                             if f.get('vcodec') != 'none':
3539                                 f['stretched_ratio'] = ratio
3540                         break
3541         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3542         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3543         if thumbnail_url:
3544             thumbnails.append({
3545                 'url': thumbnail_url,
3546             })
3547         original_thumbnails = thumbnails.copy()
3548
3549         # The best resolution thumbnails sometimes does not appear in the webpage
3550         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3551         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3552         thumbnail_names = [
3553             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3554             # in resolution, these are not the custom thumbnail. So de-prioritize them
3555             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3556             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3557         ]
3558         n_thumbnail_names = len(thumbnail_names)
3559         thumbnails.extend({
3560             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3561                 video_id=video_id, name=name, ext=ext,
3562                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3563         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3564         for thumb in thumbnails:
3565             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3566             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3567         self._remove_duplicate_formats(thumbnails)
3568         self._downloader._sort_thumbnails(original_thumbnails)
3569
3570         category = get_first(microformats, 'category') or search_meta('genre')
3571         channel_id = str_or_none(
3572             get_first(video_details, 'channelId')
3573             or get_first(microformats, 'externalChannelId')
3574             or search_meta('channelId'))
3575         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3576
3577         live_content = get_first(video_details, 'isLiveContent')
3578         is_upcoming = get_first(video_details, 'isUpcoming')
3579         if is_live is None:
3580             if is_upcoming or live_content is False:
3581                 is_live = False
3582         if is_upcoming is None and (live_content or is_live):
3583             is_upcoming = False
3584         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3585         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3586         if not duration and live_end_time and live_start_time:
3587             duration = live_end_time - live_start_time
3588
3589         if is_live and self.get_param('live_from_start'):
3590             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3591
3592         formats.extend(self._extract_storyboard(player_responses, duration))
3593
3594         # source_preference is lower for throttled/potentially damaged formats
3595         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3596
3597         info = {
3598             'id': video_id,
3599             'title': video_title,
3600             'formats': formats,
3601             'thumbnails': thumbnails,
3602             # The best thumbnail that we are sure exists. Prevents unnecessary
3603             # URL checking if user don't care about getting the best possible thumbnail
3604             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3605             'description': video_description,
3606             'uploader': get_first(video_details, 'author'),
3607             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3608             'uploader_url': owner_profile_url,
3609             'channel_id': channel_id,
3610             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3611             'duration': duration,
3612             'view_count': int_or_none(
3613                 get_first((video_details, microformats), (..., 'viewCount'))
3614                 or search_meta('interactionCount')),
3615             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3616             'age_limit': 18 if (
3617                 get_first(microformats, 'isFamilySafe') is False
3618                 or search_meta('isFamilyFriendly') == 'false'
3619                 or search_meta('og:restrictions:age') == '18+') else 0,
3620             'webpage_url': webpage_url,
3621             'categories': [category] if category else None,
3622             'tags': keywords,
3623             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3624             'is_live': is_live,
3625             'was_live': (False if is_live or is_upcoming or live_content is False
3626                          else None if is_live is None or is_upcoming is None
3627                          else live_content),
3628             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3629             'release_timestamp': live_start_time,
3630         }
3631
3632         if get_first(video_details, 'isPostLiveDvr'):
3633             self.write_debug('Video is in Post-Live Manifestless mode')
3634             info['live_status'] = 'post_live'
3635             if (duration or 0) > 4 * 3600:
3636                 self.report_warning(
3637                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3638                     'This is a known issue and patches are welcome')
3639
3640         subtitles = {}
3641         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3642         if pctr:
3643             def get_lang_code(track):
3644                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3645                         or track.get('languageCode'))
3646
3647             # Converted into dicts to remove duplicates
3648             captions = {
3649                 get_lang_code(sub): sub
3650                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3651             translation_languages = {
3652                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3653                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3654
3655             def process_language(container, base_url, lang_code, sub_name, query):
3656                 lang_subs = container.setdefault(lang_code, [])
3657                 for fmt in self._SUBTITLE_FORMATS:
3658                     query.update({
3659                         'fmt': fmt,
3660                     })
3661                     lang_subs.append({
3662                         'ext': fmt,
3663                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3664                         'name': sub_name,
3665                     })
3666
3667             # NB: Constructing the full subtitle dictionary is slow
3668             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
3669                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
3670             for lang_code, caption_track in captions.items():
3671                 base_url = caption_track.get('baseUrl')
3672                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3673                 if not base_url:
3674                     continue
3675                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3676                 if caption_track.get('kind') != 'asr':
3677                     if not lang_code:
3678                         continue
3679                     process_language(
3680                         subtitles, base_url, lang_code, lang_name, {})
3681                     if not caption_track.get('isTranslatable'):
3682                         continue
3683                 for trans_code, trans_name in translation_languages.items():
3684                     if not trans_code:
3685                         continue
3686                     orig_trans_code = trans_code
3687                     if caption_track.get('kind') != 'asr':
3688                         if not get_translated_subs:
3689                             continue
3690                         trans_code += f'-{lang_code}'
3691                         trans_name += format_field(lang_name, None, ' from %s')
3692                     # Add an "-orig" label to the original language so that it can be distinguished.
3693                     # The subs are returned without "-orig" as well for compatibility
3694                     if lang_code == f'a-{orig_trans_code}':
3695                         process_language(
3696                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3697                     # Setting tlang=lang returns damaged subtitles.
3698                     process_language(automatic_captions, base_url, trans_code, trans_name,
3699                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3700
3701         info['automatic_captions'] = automatic_captions
3702         info['subtitles'] = subtitles
3703
3704         parsed_url = urllib.parse.urlparse(url)
3705         for component in [parsed_url.fragment, parsed_url.query]:
3706             query = urllib.parse.parse_qs(component)
3707             for k, v in query.items():
3708                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3709                     d_k += '_time'
3710                     if d_k not in info and k in s_ks:
3711                         info[d_k] = parse_duration(query[k][0])
3712
3713         # Youtube Music Auto-generated description
3714         if video_description:
3715             mobj = re.search(
3716                 r'''(?xs)
3717                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3718                     (?P<album>[^\n]+)
3719                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3720                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3721                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3722                     .+\nAuto-generated\ by\ YouTube\.\s*$
3723                 ''', video_description)
3724             if mobj:
3725                 release_year = mobj.group('release_year')
3726                 release_date = mobj.group('release_date')
3727                 if release_date:
3728                     release_date = release_date.replace('-', '')
3729                     if not release_year:
3730                         release_year = release_date[:4]
3731                 info.update({
3732                     'album': mobj.group('album'.strip()),
3733                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3734                     'track': mobj.group('track').strip(),
3735                     'release_date': release_date,
3736                     'release_year': int_or_none(release_year),
3737                 })
3738
3739         initial_data = None
3740         if webpage:
3741             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
3742         if not initial_data:
3743             query = {'videoId': video_id}
3744             query.update(self._get_checkok_params())
3745             initial_data = self._extract_response(
3746                 item_id=video_id, ep='next', fatal=False,
3747                 ytcfg=master_ytcfg, query=query,
3748                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3749                 note='Downloading initial data API JSON')
3750
3751         info['comment_count'] = traverse_obj(initial_data, (
3752             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
3753             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
3754         ), (
3755             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
3756             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
3757         ), expected_type=int_or_none, get_all=False)
3758
3759         try:  # This will error if there is no livechat
3760             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3761         except (KeyError, IndexError, TypeError):
3762             pass
3763         else:
3764             info.setdefault('subtitles', {})['live_chat'] = [{
3765                 # url is needed to set cookies
3766                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
3767                 'video_id': video_id,
3768                 'ext': 'json',
3769                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3770             }]
3771
3772         if initial_data:
3773             info['chapters'] = (
3774                 self._extract_chapters_from_json(initial_data, duration)
3775                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3776                 or self._extract_chapters_from_description(video_description, duration)
3777                 or None)
3778
3779         contents = traverse_obj(
3780             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3781             expected_type=list, default=[])
3782
3783         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3784         if vpir:
3785             stl = vpir.get('superTitleLink')
3786             if stl:
3787                 stl = self._get_text(stl)
3788                 if try_get(
3789                         vpir,
3790                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3791                     info['location'] = stl
3792                 else:
3793                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3794                     if mobj:
3795                         info.update({
3796                             'series': mobj.group(1),
3797                             'season_number': int(mobj.group(2)),
3798                             'episode_number': int(mobj.group(3)),
3799                         })
3800             for tlb in (try_get(
3801                     vpir,
3802                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3803                     list) or []):
3804                 tbr = tlb.get('toggleButtonRenderer') or {}
3805                 for getter, regex in [(
3806                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3807                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3808                             lambda x: x['accessibility'],
3809                             lambda x: x['accessibilityData']['accessibilityData'],
3810                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3811                     label = (try_get(tbr, getter, dict) or {}).get('label')
3812                     if label:
3813                         mobj = re.match(regex, label)
3814                         if mobj:
3815                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3816                             break
3817             sbr_tooltip = try_get(
3818                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3819             if sbr_tooltip:
3820                 like_count, dislike_count = sbr_tooltip.split(' / ')
3821                 info.update({
3822                     'like_count': str_to_int(like_count),
3823                     'dislike_count': str_to_int(dislike_count),
3824                 })
3825         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3826         if vsir:
3827             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3828             info.update({
3829                 'channel': self._get_text(vor, 'title'),
3830                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3831
3832             rows = try_get(
3833                 vsir,
3834                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3835                 list) or []
3836             multiple_songs = False
3837             for row in rows:
3838                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3839                     multiple_songs = True
3840                     break
3841             for row in rows:
3842                 mrr = row.get('metadataRowRenderer') or {}
3843                 mrr_title = mrr.get('title')
3844                 if not mrr_title:
3845                     continue
3846                 mrr_title = self._get_text(mrr, 'title')
3847                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3848                 if mrr_title == 'License':
3849                     info['license'] = mrr_contents_text
3850                 elif not multiple_songs:
3851                     if mrr_title == 'Album':
3852                         info['album'] = mrr_contents_text
3853                     elif mrr_title == 'Artist':
3854                         info['artist'] = mrr_contents_text
3855                     elif mrr_title == 'Song':
3856                         info['track'] = mrr_contents_text
3857
3858         fallbacks = {
3859             'channel': 'uploader',
3860             'channel_id': 'uploader_id',
3861             'channel_url': 'uploader_url',
3862         }
3863
3864         # The upload date for scheduled, live and past live streams / premieres in microformats
3865         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3866         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3867         upload_date = (
3868             unified_strdate(get_first(microformats, 'uploadDate'))
3869             or unified_strdate(search_meta('uploadDate')))
3870         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3871             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3872         info['upload_date'] = upload_date
3873
3874         for to, frm in fallbacks.items():
3875             if not info.get(to):
3876                 info[to] = info.get(frm)
3877
3878         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3879             v = info.get(s_k)
3880             if v:
3881                 info[d_k] = v
3882
3883         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3884         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3885         is_membersonly = None
3886         is_premium = None
3887         if initial_data and is_private is not None:
3888             is_membersonly = False
3889             is_premium = False
3890             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3891             badge_labels = set()
3892             for content in contents:
3893                 if not isinstance(content, dict):
3894                     continue
3895                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3896             for badge_label in badge_labels:
3897                 if badge_label.lower() == 'members only':
3898                     is_membersonly = True
3899                 elif badge_label.lower() == 'premium':
3900                     is_premium = True
3901                 elif badge_label.lower() == 'unlisted':
3902                     is_unlisted = True
3903
3904         info['availability'] = self._availability(
3905             is_private=is_private,
3906             needs_premium=is_premium,
3907             needs_subscription=is_membersonly,
3908             needs_auth=info['age_limit'] >= 18,
3909             is_unlisted=None if is_private is None else is_unlisted)
3910
3911         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3912
3913         self.mark_watched(video_id, player_responses)
3914
3915         return info
3916
3917
3918 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3919
3920     @staticmethod
3921     def passthrough_smuggled_data(func):
3922         def _smuggle(entries, smuggled_data):
3923             for entry in entries:
3924                 # TODO: Convert URL to music.youtube instead.
3925                 # Do we need to passthrough any other smuggled_data?
3926                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3927                 yield entry
3928
3929         @functools.wraps(func)
3930         def wrapper(self, url):
3931             url, smuggled_data = unsmuggle_url(url, {})
3932             if self.is_music_url(url):
3933                 smuggled_data['is_music_url'] = True
3934             info_dict = func(self, url, smuggled_data)
3935             if smuggled_data and info_dict.get('entries'):
3936                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3937             return info_dict
3938         return wrapper
3939
3940     def _extract_channel_id(self, webpage):
3941         channel_id = self._html_search_meta(
3942             'channelId', webpage, 'channel id', default=None)
3943         if channel_id:
3944             return channel_id
3945         channel_url = self._html_search_meta(
3946             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3947              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3948              'twitter:app:url:googleplay'), webpage, 'channel url')
3949         return self._search_regex(
3950             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3951             channel_url, 'channel id')
3952
3953     @staticmethod
3954     def _extract_basic_item_renderer(item):
3955         # Modified from _extract_grid_item_renderer
3956         known_basic_renderers = (
3957             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3958         )
3959         for key, renderer in item.items():
3960             if not isinstance(renderer, dict):
3961                 continue
3962             elif key in known_basic_renderers:
3963                 return renderer
3964             elif key.startswith('grid') and key.endswith('Renderer'):
3965                 return renderer
3966
3967     def _grid_entries(self, grid_renderer):
3968         for item in grid_renderer['items']:
3969             if not isinstance(item, dict):
3970                 continue
3971             renderer = self._extract_basic_item_renderer(item)
3972             if not isinstance(renderer, dict):
3973                 continue
3974             title = self._get_text(renderer, 'title')
3975
3976             # playlist
3977             playlist_id = renderer.get('playlistId')
3978             if playlist_id:
3979                 yield self.url_result(
3980                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3981                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3982                     video_title=title)
3983                 continue
3984             # video
3985             video_id = renderer.get('videoId')
3986             if video_id:
3987                 yield self._extract_video(renderer)
3988                 continue
3989             # channel
3990             channel_id = renderer.get('channelId')
3991             if channel_id:
3992                 yield self.url_result(
3993                     'https://www.youtube.com/channel/%s' % channel_id,
3994                     ie=YoutubeTabIE.ie_key(), video_title=title)
3995                 continue
3996             # generic endpoint URL support
3997             ep_url = urljoin('https://www.youtube.com/', try_get(
3998                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3999                 str))
4000             if ep_url:
4001                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4002                     if ie.suitable(ep_url):
4003                         yield self.url_result(
4004                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4005                         break
4006
4007     def _music_reponsive_list_entry(self, renderer):
4008         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4009         if video_id:
4010             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4011                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4012         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4013         if playlist_id:
4014             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4015             if video_id:
4016                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4017                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4018             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4019                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4020         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4021         if browse_id:
4022             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4023                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4024
4025     def _shelf_entries_from_content(self, shelf_renderer):
4026         content = shelf_renderer.get('content')
4027         if not isinstance(content, dict):
4028             return
4029         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4030         if renderer:
4031             # TODO: add support for nested playlists so each shelf is processed
4032             # as separate playlist
4033             # TODO: this includes only first N items
4034             yield from self._grid_entries(renderer)
4035         renderer = content.get('horizontalListRenderer')
4036         if renderer:
4037             # TODO
4038             pass
4039
4040     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4041         ep = try_get(
4042             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4043             str)
4044         shelf_url = urljoin('https://www.youtube.com', ep)
4045         if shelf_url:
4046             # Skipping links to another channels, note that checking for
4047             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4048             # will not work
4049             if skip_channels and '/channels?' in shelf_url:
4050                 return
4051             title = self._get_text(shelf_renderer, 'title')
4052             yield self.url_result(shelf_url, video_title=title)
4053         # Shelf may not contain shelf URL, fallback to extraction from content
4054         yield from self._shelf_entries_from_content(shelf_renderer)
4055
4056     def _playlist_entries(self, video_list_renderer):
4057         for content in video_list_renderer['contents']:
4058             if not isinstance(content, dict):
4059                 continue
4060             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4061             if not isinstance(renderer, dict):
4062                 continue
4063             video_id = renderer.get('videoId')
4064             if not video_id:
4065                 continue
4066             yield self._extract_video(renderer)
4067
4068     def _rich_entries(self, rich_grid_renderer):
4069         renderer = try_get(
4070             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
4071         video_id = renderer.get('videoId')
4072         if not video_id:
4073             return
4074         yield self._extract_video(renderer)
4075
4076     def _video_entry(self, video_renderer):
4077         video_id = video_renderer.get('videoId')
4078         if video_id:
4079             return self._extract_video(video_renderer)
4080
4081     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4082         url = urljoin('https://youtube.com', traverse_obj(
4083             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4084         if url:
4085             return self.url_result(
4086                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4087
4088     def _post_thread_entries(self, post_thread_renderer):
4089         post_renderer = try_get(
4090             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4091         if not post_renderer:
4092             return
4093         # video attachment
4094         video_renderer = try_get(
4095             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4096         video_id = video_renderer.get('videoId')
4097         if video_id:
4098             entry = self._extract_video(video_renderer)
4099             if entry:
4100                 yield entry
4101         # playlist attachment
4102         playlist_id = try_get(
4103             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4104         if playlist_id:
4105             yield self.url_result(
4106                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4107                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4108         # inline video links
4109         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4110         for run in runs:
4111             if not isinstance(run, dict):
4112                 continue
4113             ep_url = try_get(
4114                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4115             if not ep_url:
4116                 continue
4117             if not YoutubeIE.suitable(ep_url):
4118                 continue
4119             ep_video_id = YoutubeIE._match_id(ep_url)
4120             if video_id == ep_video_id:
4121                 continue
4122             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4123
4124     def _post_thread_continuation_entries(self, post_thread_continuation):
4125         contents = post_thread_continuation.get('contents')
4126         if not isinstance(contents, list):
4127             return
4128         for content in contents:
4129             renderer = content.get('backstagePostThreadRenderer')
4130             if isinstance(renderer, dict):
4131                 yield from self._post_thread_entries(renderer)
4132                 continue
4133             renderer = content.get('videoRenderer')
4134             if isinstance(renderer, dict):
4135                 yield self._video_entry(renderer)
4136
4137     r''' # unused
4138     def _rich_grid_entries(self, contents):
4139         for content in contents:
4140             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4141             if video_renderer:
4142                 entry = self._video_entry(video_renderer)
4143                 if entry:
4144                     yield entry
4145     '''
4146
4147     def _extract_entries(self, parent_renderer, continuation_list):
4148         # continuation_list is modified in-place with continuation_list = [continuation_token]
4149         continuation_list[:] = [None]
4150         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4151         for content in contents:
4152             if not isinstance(content, dict):
4153                 continue
4154             is_renderer = traverse_obj(
4155                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4156                 expected_type=dict)
4157             if not is_renderer:
4158                 renderer = content.get('richItemRenderer')
4159                 if renderer:
4160                     for entry in self._rich_entries(renderer):
4161                         yield entry
4162                     continuation_list[0] = self._extract_continuation(parent_renderer)
4163                 continue
4164             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4165             for isr_content in isr_contents:
4166                 if not isinstance(isr_content, dict):
4167                     continue
4168
4169                 known_renderers = {
4170                     'playlistVideoListRenderer': self._playlist_entries,
4171                     'gridRenderer': self._grid_entries,
4172                     'reelShelfRenderer': self._grid_entries,
4173                     'shelfRenderer': self._shelf_entries,
4174                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4175                     'backstagePostThreadRenderer': self._post_thread_entries,
4176                     'videoRenderer': lambda x: [self._video_entry(x)],
4177                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4178                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4179                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4180                 }
4181                 for key, renderer in isr_content.items():
4182                     if key not in known_renderers:
4183                         continue
4184                     for entry in known_renderers[key](renderer):
4185                         if entry:
4186                             yield entry
4187                     continuation_list[0] = self._extract_continuation(renderer)
4188                     break
4189
4190             if not continuation_list[0]:
4191                 continuation_list[0] = self._extract_continuation(is_renderer)
4192
4193         if not continuation_list[0]:
4194             continuation_list[0] = self._extract_continuation(parent_renderer)
4195
4196     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4197         continuation_list = [None]
4198         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4199         tab_content = try_get(tab, lambda x: x['content'], dict)
4200         if not tab_content:
4201             return
4202         parent_renderer = (
4203             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4204             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4205         yield from extract_entries(parent_renderer)
4206         continuation = continuation_list[0]
4207
4208         for page_num in itertools.count(1):
4209             if not continuation:
4210                 break
4211             headers = self.generate_api_headers(
4212                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4213             response = self._extract_response(
4214                 item_id=f'{item_id} page {page_num}',
4215                 query=continuation, headers=headers, ytcfg=ytcfg,
4216                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4217
4218             if not response:
4219                 break
4220             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4221             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4222             visitor_data = self._extract_visitor_data(response) or visitor_data
4223
4224             known_continuation_renderers = {
4225                 'playlistVideoListContinuation': self._playlist_entries,
4226                 'gridContinuation': self._grid_entries,
4227                 'itemSectionContinuation': self._post_thread_continuation_entries,
4228                 'sectionListContinuation': extract_entries,  # for feeds
4229             }
4230             continuation_contents = try_get(
4231                 response, lambda x: x['continuationContents'], dict) or {}
4232             continuation_renderer = None
4233             for key, value in continuation_contents.items():
4234                 if key not in known_continuation_renderers:
4235                     continue
4236                 continuation_renderer = value
4237                 continuation_list = [None]
4238                 yield from known_continuation_renderers[key](continuation_renderer)
4239                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4240                 break
4241             if continuation_renderer:
4242                 continue
4243
4244             known_renderers = {
4245                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4246                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4247                 'gridVideoRenderer': (self._grid_entries, 'items'),
4248                 'gridChannelRenderer': (self._grid_entries, 'items'),
4249                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4250                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4251                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4252                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4253             }
4254             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4255             continuation_items = try_get(
4256                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4257             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4258             video_items_renderer = None
4259             for key, value in continuation_item.items():
4260                 if key not in known_renderers:
4261                     continue
4262                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4263                 continuation_list = [None]
4264                 yield from known_renderers[key][0](video_items_renderer)
4265                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4266                 break
4267             if video_items_renderer:
4268                 continue
4269             break
4270
4271     @staticmethod
4272     def _extract_selected_tab(tabs, fatal=True):
4273         for tab in tabs:
4274             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4275             if renderer.get('selected') is True:
4276                 return renderer
4277         else:
4278             if fatal:
4279                 raise ExtractorError('Unable to find selected tab')
4280
4281     def _extract_uploader(self, data):
4282         uploader = {}
4283         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4284         owner = try_get(
4285             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4286         if owner:
4287             owner_text = owner.get('text')
4288             uploader['uploader'] = self._search_regex(
4289                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4290             uploader['uploader_id'] = try_get(
4291                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], str)
4292             uploader['uploader_url'] = urljoin(
4293                 'https://www.youtube.com/',
4294                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], str))
4295         return {k: v for k, v in uploader.items() if v is not None}
4296
4297     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4298         playlist_id = title = description = channel_url = channel_name = channel_id = None
4299         tags = []
4300
4301         selected_tab = self._extract_selected_tab(tabs)
4302         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4303         renderer = try_get(
4304             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4305         if renderer:
4306             channel_name = renderer.get('title')
4307             channel_url = renderer.get('channelUrl')
4308             channel_id = renderer.get('externalId')
4309         else:
4310             renderer = try_get(
4311                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4312
4313         if renderer:
4314             title = renderer.get('title')
4315             description = renderer.get('description', '')
4316             playlist_id = channel_id
4317             tags = renderer.get('keywords', '').split()
4318
4319         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4320         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4321         def _get_uncropped(url):
4322             return url_or_none((url or '').split('=')[0] + '=s0')
4323
4324         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4325         if avatar_thumbnails:
4326             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4327             if uncropped_avatar:
4328                 avatar_thumbnails.append({
4329                     'url': uncropped_avatar,
4330                     'id': 'avatar_uncropped',
4331                     'preference': 1
4332                 })
4333
4334         channel_banners = self._extract_thumbnails(
4335             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4336         for banner in channel_banners:
4337             banner['preference'] = -10
4338
4339         if channel_banners:
4340             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4341             if uncropped_banner:
4342                 channel_banners.append({
4343                     'url': uncropped_banner,
4344                     'id': 'banner_uncropped',
4345                     'preference': -5
4346                 })
4347
4348         primary_thumbnails = self._extract_thumbnails(
4349             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4350
4351         if playlist_id is None:
4352             playlist_id = item_id
4353
4354         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4355         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4356         if title is None:
4357             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4358         title += format_field(selected_tab, 'title', ' - %s')
4359         title += format_field(selected_tab, 'expandedText', ' - %s')
4360
4361         metadata = {
4362             'playlist_id': playlist_id,
4363             'playlist_title': title,
4364             'playlist_description': description,
4365             'uploader': channel_name,
4366             'uploader_id': channel_id,
4367             'uploader_url': channel_url,
4368             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4369             'tags': tags,
4370             'view_count': self._get_count(playlist_stats, 1),
4371             'availability': self._extract_availability(data),
4372             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4373             'playlist_count': self._get_count(playlist_stats, 0),
4374             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4375         }
4376         if not channel_id:
4377             metadata.update(self._extract_uploader(data))
4378         metadata.update({
4379             'channel': metadata['uploader'],
4380             'channel_id': metadata['uploader_id'],
4381             'channel_url': metadata['uploader_url']})
4382         return self.playlist_result(
4383             self._entries(
4384                 selected_tab, playlist_id, ytcfg,
4385                 self._extract_account_syncid(ytcfg, data),
4386                 self._extract_visitor_data(data, ytcfg)),
4387             **metadata)
4388
4389     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4390         first_id = last_id = response = None
4391         for page_num in itertools.count(1):
4392             videos = list(self._playlist_entries(playlist))
4393             if not videos:
4394                 return
4395             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4396             if start >= len(videos):
4397                 return
4398             yield from videos[start:]
4399             first_id = first_id or videos[0]['id']
4400             last_id = videos[-1]['id']
4401             watch_endpoint = try_get(
4402                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4403             headers = self.generate_api_headers(
4404                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4405                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4406             query = {
4407                 'playlistId': playlist_id,
4408                 'videoId': watch_endpoint.get('videoId') or last_id,
4409                 'index': watch_endpoint.get('index') or len(videos),
4410                 'params': watch_endpoint.get('params') or 'OAE%3D'
4411             }
4412             response = self._extract_response(
4413                 item_id='%s page %d' % (playlist_id, page_num),
4414                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4415                 check_get_keys='contents'
4416             )
4417             playlist = try_get(
4418                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4419
4420     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4421         title = playlist.get('title') or try_get(
4422             data, lambda x: x['titleText']['simpleText'], str)
4423         playlist_id = playlist.get('playlistId') or item_id
4424
4425         # Delegating everything except mix playlists to regular tab-based playlist URL
4426         playlist_url = urljoin(url, try_get(
4427             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4428             str))
4429
4430         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4431         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4432         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4433
4434         if playlist_url and playlist_url != url and not is_known_unviewable:
4435             return self.url_result(
4436                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4437                 video_title=title)
4438
4439         return self.playlist_result(
4440             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4441             playlist_id=playlist_id, playlist_title=title)
4442
4443     def _extract_availability(self, data):
4444         """
4445         Gets the availability of a given playlist/tab.
4446         Note: Unless YouTube tells us explicitly, we do not assume it is public
4447         @param data: response
4448         """
4449         is_private = is_unlisted = None
4450         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4451         badge_labels = self._extract_badges(renderer)
4452
4453         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4454         privacy_dropdown_entries = try_get(
4455             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4456         for renderer_dict in privacy_dropdown_entries:
4457             is_selected = try_get(
4458                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4459             if not is_selected:
4460                 continue
4461             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4462             if label:
4463                 badge_labels.add(label.lower())
4464                 break
4465
4466         for badge_label in badge_labels:
4467             if badge_label == 'unlisted':
4468                 is_unlisted = True
4469             elif badge_label == 'private':
4470                 is_private = True
4471             elif badge_label == 'public':
4472                 is_unlisted = is_private = False
4473         return self._availability(is_private, False, False, False, is_unlisted)
4474
4475     @staticmethod
4476     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4477         sidebar_renderer = try_get(
4478             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4479         for item in sidebar_renderer:
4480             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4481             if renderer:
4482                 return renderer
4483
4484     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4485         """
4486         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4487         """
4488         browse_id = params = None
4489         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4490         if not renderer:
4491             return
4492         menu_renderer = try_get(
4493             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4494         for menu_item in menu_renderer:
4495             if not isinstance(menu_item, dict):
4496                 continue
4497             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4498             text = try_get(
4499                 nav_item_renderer, lambda x: x['text']['simpleText'], str)
4500             if not text or text.lower() != 'show unavailable videos':
4501                 continue
4502             browse_endpoint = try_get(
4503                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4504             browse_id = browse_endpoint.get('browseId')
4505             params = browse_endpoint.get('params')
4506             break
4507
4508         headers = self.generate_api_headers(
4509             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4510             visitor_data=self._extract_visitor_data(data, ytcfg))
4511         query = {
4512             'params': params or 'wgYCCAA=',
4513             'browseId': browse_id or 'VL%s' % item_id
4514         }
4515         return self._extract_response(
4516             item_id=item_id, headers=headers, query=query,
4517             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4518             note='Downloading API JSON with unavailable videos')
4519
4520     @functools.cached_property
4521     def skip_webpage(self):
4522         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4523
4524     def _extract_webpage(self, url, item_id, fatal=True):
4525         retries = self.get_param('extractor_retries', 3)
4526         count = -1
4527         webpage = data = last_error = None
4528         while count < retries:
4529             count += 1
4530             # Sometimes youtube returns a webpage with incomplete ytInitialData
4531             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4532             if last_error:
4533                 self.report_warning('%s. Retrying ...' % last_error)
4534             try:
4535                 webpage = self._download_webpage(
4536                     url, item_id,
4537                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4538                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4539             except ExtractorError as e:
4540                 if isinstance(e.cause, network_exceptions):
4541                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4542                         last_error = error_to_compat_str(e.cause or e.msg)
4543                         if count < retries:
4544                             continue
4545                 if fatal:
4546                     raise
4547                 self.report_warning(error_to_compat_str(e))
4548                 break
4549             else:
4550                 try:
4551                     self._extract_and_report_alerts(data)
4552                 except ExtractorError as e:
4553                     if fatal:
4554                         raise
4555                     self.report_warning(error_to_compat_str(e))
4556                     break
4557
4558                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4559                     break
4560
4561                 last_error = 'Incomplete yt initial data received'
4562                 if count >= retries:
4563                     if fatal:
4564                         raise ExtractorError(last_error)
4565                     self.report_warning(last_error)
4566                     break
4567
4568         return webpage, data
4569
4570     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4571         """Use if failed to extract ytcfg (and data) from initial webpage"""
4572         if not ytcfg and self.is_authenticated:
4573             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4574             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4575                 raise ExtractorError(
4576                     f'{msg}. If you are not downloading private content, or '
4577                     'your cookies are only for the first account and channel,'
4578                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4579                     expected=True)
4580             self.report_warning(msg, only_once=True)
4581
4582     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4583         data = None
4584         if not self.skip_webpage:
4585             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4586             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4587             # Reject webpage data if redirected to home page without explicitly requesting
4588             selected_tab = self._extract_selected_tab(traverse_obj(
4589                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4590             if (url != 'https://www.youtube.com/feed/recommended'
4591                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4592                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4593                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4594                 if fatal:
4595                     raise ExtractorError(msg, expected=True)
4596                 self.report_warning(msg, only_once=True)
4597         if not data:
4598             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4599             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4600         return data, ytcfg
4601
4602     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4603         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4604         resolve_response = self._extract_response(
4605             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4606             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4607         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4608         for ep_key, ep in endpoints.items():
4609             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4610             if params:
4611                 return self._extract_response(
4612                     item_id=item_id, query=params, ep=ep, headers=headers,
4613                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4614                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4615         err_note = 'Failed to resolve url (does the playlist exist?)'
4616         if fatal:
4617             raise ExtractorError(err_note, expected=True)
4618         self.report_warning(err_note, item_id)
4619
4620     _SEARCH_PARAMS = None
4621
4622     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4623         data = {'query': query}
4624         if params is NO_DEFAULT:
4625             params = self._SEARCH_PARAMS
4626         if params:
4627             data['params'] = params
4628
4629         content_keys = (
4630             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4631             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4632             # ytmusic search
4633             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4634             ('continuationContents', ),
4635         )
4636         display_id = f'query "{query}"'
4637         check_get_keys = tuple({keys[0] for keys in content_keys})
4638         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4639         self._report_playlist_authcheck(ytcfg, fatal=False)
4640
4641         continuation_list = [None]
4642         search = None
4643         for page_num in itertools.count(1):
4644             data.update(continuation_list[0] or {})
4645             headers = self.generate_api_headers(
4646                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4647             search = self._extract_response(
4648                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4649                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4650             slr_contents = traverse_obj(search, *content_keys)
4651             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4652             if not continuation_list[0]:
4653                 break
4654
4655
4656 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4657     IE_DESC = 'YouTube Tabs'
4658     _VALID_URL = r'''(?x:
4659         https?://
4660             (?:\w+\.)?
4661             (?:
4662                 youtube(?:kids)?\.com|
4663                 %(invidious)s
4664             )/
4665             (?:
4666                 (?P<channel_type>channel|c|user|browse)/|
4667                 (?P<not_channel>
4668                     feed/|hashtag/|
4669                     (?:playlist|watch)\?.*?\blist=
4670                 )|
4671                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4672             )
4673             (?P<id>[^/?\#&]+)
4674     )''' % {
4675         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4676         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4677     }
4678     IE_NAME = 'youtube:tab'
4679
4680     _TESTS = [{
4681         'note': 'playlists, multipage',
4682         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4683         'playlist_mincount': 94,
4684         'info_dict': {
4685             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4686             'title': 'Igor Kleiner - Playlists',
4687             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4688             'uploader': 'Igor Kleiner',
4689             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4690             'channel': 'Igor Kleiner',
4691             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4692             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4693             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4694             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4695             'channel_follower_count': int
4696         },
4697     }, {
4698         'note': 'playlists, multipage, different order',
4699         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4700         'playlist_mincount': 94,
4701         'info_dict': {
4702             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4703             'title': 'Igor Kleiner - Playlists',
4704             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4705             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4706             'uploader': 'Igor Kleiner',
4707             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4708             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4709             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4710             'channel': 'Igor Kleiner',
4711             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4712             'channel_follower_count': int
4713         },
4714     }, {
4715         'note': 'playlists, series',
4716         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4717         'playlist_mincount': 5,
4718         'info_dict': {
4719             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4720             'title': '3Blue1Brown - Playlists',
4721             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4722             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4723             'uploader': '3Blue1Brown',
4724             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4725             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4726             'channel': '3Blue1Brown',
4727             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4728             'tags': ['Mathematics'],
4729             'channel_follower_count': int
4730         },
4731     }, {
4732         'note': 'playlists, singlepage',
4733         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4734         'playlist_mincount': 4,
4735         'info_dict': {
4736             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4737             'title': 'ThirstForScience - Playlists',
4738             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4739             'uploader': 'ThirstForScience',
4740             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4741             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4742             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4743             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4744             'tags': 'count:13',
4745             'channel': 'ThirstForScience',
4746             'channel_follower_count': int
4747         }
4748     }, {
4749         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4750         'only_matching': True,
4751     }, {
4752         'note': 'basic, single video playlist',
4753         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4754         'info_dict': {
4755             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4756             'uploader': 'Sergey M.',
4757             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4758             'title': 'youtube-dl public playlist',
4759             'description': '',
4760             'tags': [],
4761             'view_count': int,
4762             'modified_date': '20201130',
4763             'channel': 'Sergey M.',
4764             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4765             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4766             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4767         },
4768         'playlist_count': 1,
4769     }, {
4770         'note': 'empty playlist',
4771         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4772         'info_dict': {
4773             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4774             'uploader': 'Sergey M.',
4775             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4776             'title': 'youtube-dl empty playlist',
4777             'tags': [],
4778             'channel': 'Sergey M.',
4779             'description': '',
4780             'modified_date': '20160902',
4781             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4782             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4783             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4784         },
4785         'playlist_count': 0,
4786     }, {
4787         'note': 'Home tab',
4788         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4789         'info_dict': {
4790             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4791             'title': 'lex will - Home',
4792             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4793             'uploader': 'lex will',
4794             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4795             'channel': 'lex will',
4796             'tags': ['bible', 'history', 'prophesy'],
4797             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4798             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4799             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4800             'channel_follower_count': int
4801         },
4802         'playlist_mincount': 2,
4803     }, {
4804         'note': 'Videos tab',
4805         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4806         'info_dict': {
4807             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4808             'title': 'lex will - Videos',
4809             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4810             'uploader': 'lex will',
4811             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4812             'tags': ['bible', 'history', 'prophesy'],
4813             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4814             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4815             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4816             'channel': 'lex will',
4817             'channel_follower_count': int
4818         },
4819         'playlist_mincount': 975,
4820     }, {
4821         'note': 'Videos tab, sorted by popular',
4822         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4823         'info_dict': {
4824             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4825             'title': 'lex will - Videos',
4826             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4827             'uploader': 'lex will',
4828             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4829             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4830             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4831             'channel': 'lex will',
4832             'tags': ['bible', 'history', 'prophesy'],
4833             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4834             'channel_follower_count': int
4835         },
4836         'playlist_mincount': 199,
4837     }, {
4838         'note': 'Playlists tab',
4839         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4840         'info_dict': {
4841             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4842             'title': 'lex will - Playlists',
4843             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4844             'uploader': 'lex will',
4845             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4846             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4847             'channel': 'lex will',
4848             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4849             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4850             'tags': ['bible', 'history', 'prophesy'],
4851             'channel_follower_count': int
4852         },
4853         'playlist_mincount': 17,
4854     }, {
4855         'note': 'Community tab',
4856         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4857         'info_dict': {
4858             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4859             'title': 'lex will - Community',
4860             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4861             'uploader': 'lex will',
4862             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4863             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4864             'channel': 'lex will',
4865             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4866             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4867             'tags': ['bible', 'history', 'prophesy'],
4868             'channel_follower_count': int
4869         },
4870         'playlist_mincount': 18,
4871     }, {
4872         'note': 'Channels tab',
4873         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4874         'info_dict': {
4875             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4876             'title': 'lex will - Channels',
4877             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4878             'uploader': 'lex will',
4879             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4880             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4881             'channel': 'lex will',
4882             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4883             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4884             'tags': ['bible', 'history', 'prophesy'],
4885             'channel_follower_count': int
4886         },
4887         'playlist_mincount': 12,
4888     }, {
4889         'note': 'Search tab',
4890         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4891         'playlist_mincount': 40,
4892         'info_dict': {
4893             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4894             'title': '3Blue1Brown - Search - linear algebra',
4895             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4896             'uploader': '3Blue1Brown',
4897             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4898             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4899             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4900             'tags': ['Mathematics'],
4901             'channel': '3Blue1Brown',
4902             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4903             'channel_follower_count': int
4904         },
4905     }, {
4906         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4907         'only_matching': True,
4908     }, {
4909         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4910         'only_matching': True,
4911     }, {
4912         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4913         'only_matching': True,
4914     }, {
4915         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4916         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4917         'info_dict': {
4918             'title': '29C3: Not my department',
4919             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4920             'uploader': 'Christiaan008',
4921             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4922             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4923             'tags': [],
4924             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4925             'view_count': int,
4926             'modified_date': '20150605',
4927             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4928             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4929             'channel': 'Christiaan008',
4930         },
4931         'playlist_count': 96,
4932     }, {
4933         'note': 'Large playlist',
4934         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4935         'info_dict': {
4936             'title': 'Uploads from Cauchemar',
4937             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4938             'uploader': 'Cauchemar',
4939             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4940             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4941             'tags': [],
4942             'modified_date': r're:\d{8}',
4943             'channel': 'Cauchemar',
4944             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4945             'view_count': int,
4946             'description': '',
4947             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4948         },
4949         'playlist_mincount': 1123,
4950         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4951     }, {
4952         'note': 'even larger playlist, 8832 videos',
4953         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4954         'only_matching': True,
4955     }, {
4956         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4957         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4958         'info_dict': {
4959             'title': 'Uploads from Interstellar Movie',
4960             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4961             'uploader': 'Interstellar Movie',
4962             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4963             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4964             'tags': [],
4965             'view_count': int,
4966             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4967             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4968             'channel': 'Interstellar Movie',
4969             'description': '',
4970             'modified_date': r're:\d{8}',
4971         },
4972         'playlist_mincount': 21,
4973     }, {
4974         'note': 'Playlist with "show unavailable videos" button',
4975         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4976         'info_dict': {
4977             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4978             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4979             'uploader': 'Phim Siêu Nhân Nhật Bản',
4980             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4981             'view_count': int,
4982             'channel': 'Phim Siêu Nhân Nhật Bản',
4983             'tags': [],
4984             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4985             'description': '',
4986             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4987             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4988             'modified_date': r're:\d{8}',
4989         },
4990         'playlist_mincount': 200,
4991         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4992     }, {
4993         'note': 'Playlist with unavailable videos in page 7',
4994         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4995         'info_dict': {
4996             'title': 'Uploads from BlankTV',
4997             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4998             'uploader': 'BlankTV',
4999             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5000             'channel': 'BlankTV',
5001             'channel_url': 'https://www.youtube.com/c/blanktv',
5002             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5003             'view_count': int,
5004             'tags': [],
5005             'uploader_url': 'https://www.youtube.com/c/blanktv',
5006             'modified_date': r're:\d{8}',
5007             'description': '',
5008         },
5009         'playlist_mincount': 1000,
5010         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5011     }, {
5012         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5013         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5014         'info_dict': {
5015             'title': 'Data Analysis with Dr Mike Pound',
5016             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5017             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5018             'uploader': 'Computerphile',
5019             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5020             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5021             'tags': [],
5022             'view_count': int,
5023             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5024             'channel_url': 'https://www.youtube.com/user/Computerphile',
5025             'channel': 'Computerphile',
5026         },
5027         'playlist_mincount': 11,
5028     }, {
5029         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5030         'only_matching': True,
5031     }, {
5032         'note': 'Playlist URL that does not actually serve a playlist',
5033         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5034         'info_dict': {
5035             'id': 'FqZTN594JQw',
5036             'ext': 'webm',
5037             'title': "Smiley's People 01 detective, Adventure Series, Action",
5038             'uploader': 'STREEM',
5039             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5040             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5041             'upload_date': '20150526',
5042             'license': 'Standard YouTube License',
5043             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5044             'categories': ['People & Blogs'],
5045             'tags': list,
5046             'view_count': int,
5047             'like_count': int,
5048         },
5049         'params': {
5050             'skip_download': True,
5051         },
5052         'skip': 'This video is not available.',
5053         'add_ie': [YoutubeIE.ie_key()],
5054     }, {
5055         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5056         'only_matching': True,
5057     }, {
5058         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5059         'only_matching': True,
5060     }, {
5061         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5062         'info_dict': {
5063             'id': 'Wq15eF5vCbI',  # This will keep changing
5064             'ext': 'mp4',
5065             'title': str,
5066             'uploader': 'Sky News',
5067             'uploader_id': 'skynews',
5068             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5069             'upload_date': r're:\d{8}',
5070             'description': str,
5071             'categories': ['News & Politics'],
5072             'tags': list,
5073             'like_count': int,
5074             'release_timestamp': 1642502819,
5075             'channel': 'Sky News',
5076             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5077             'age_limit': 0,
5078             'view_count': int,
5079             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5080             'playable_in_embed': True,
5081             'release_date': '20220118',
5082             'availability': 'public',
5083             'live_status': 'is_live',
5084             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5085             'channel_follower_count': int
5086         },
5087         'params': {
5088             'skip_download': True,
5089         },
5090         'expected_warnings': ['Ignoring subtitle tracks found in '],
5091     }, {
5092         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5093         'info_dict': {
5094             'id': 'a48o2S1cPoo',
5095             'ext': 'mp4',
5096             'title': 'The Young Turks - Live Main Show',
5097             'uploader': 'The Young Turks',
5098             'uploader_id': 'TheYoungTurks',
5099             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5100             'upload_date': '20150715',
5101             'license': 'Standard YouTube License',
5102             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5103             'categories': ['News & Politics'],
5104             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5105             'like_count': int,
5106         },
5107         'params': {
5108             'skip_download': True,
5109         },
5110         'only_matching': True,
5111     }, {
5112         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5113         'only_matching': True,
5114     }, {
5115         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5116         'only_matching': True,
5117     }, {
5118         'note': 'A channel that is not live. Should raise error',
5119         'url': 'https://www.youtube.com/user/numberphile/live',
5120         'only_matching': True,
5121     }, {
5122         'url': 'https://www.youtube.com/feed/trending',
5123         'only_matching': True,
5124     }, {
5125         'url': 'https://www.youtube.com/feed/library',
5126         'only_matching': True,
5127     }, {
5128         'url': 'https://www.youtube.com/feed/history',
5129         'only_matching': True,
5130     }, {
5131         'url': 'https://www.youtube.com/feed/subscriptions',
5132         'only_matching': True,
5133     }, {
5134         'url': 'https://www.youtube.com/feed/watch_later',
5135         'only_matching': True,
5136     }, {
5137         'note': 'Recommended - redirects to home page.',
5138         'url': 'https://www.youtube.com/feed/recommended',
5139         'only_matching': True,
5140     }, {
5141         'note': 'inline playlist with not always working continuations',
5142         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5143         'only_matching': True,
5144     }, {
5145         'url': 'https://www.youtube.com/course',
5146         'only_matching': True,
5147     }, {
5148         'url': 'https://www.youtube.com/zsecurity',
5149         'only_matching': True,
5150     }, {
5151         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5152         'only_matching': True,
5153     }, {
5154         'url': 'https://www.youtube.com/TheYoungTurks/live',
5155         'only_matching': True,
5156     }, {
5157         'url': 'https://www.youtube.com/hashtag/cctv9',
5158         'info_dict': {
5159             'id': 'cctv9',
5160             'title': '#cctv9',
5161             'tags': [],
5162         },
5163         'playlist_mincount': 350,
5164     }, {
5165         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5166         'only_matching': True,
5167     }, {
5168         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5169         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5170         'only_matching': True
5171     }, {
5172         'note': '/browse/ should redirect to /channel/',
5173         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5174         'only_matching': True
5175     }, {
5176         'note': 'VLPL, should redirect to playlist?list=PL...',
5177         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5178         'info_dict': {
5179             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5180             'uploader': 'NoCopyrightSounds',
5181             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5182             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5183             'title': 'NCS : All Releases 💿',
5184             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5185             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5186             'modified_date': r're:\d{8}',
5187             'view_count': int,
5188             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5189             'tags': [],
5190             'channel': 'NoCopyrightSounds',
5191         },
5192         'playlist_mincount': 166,
5193         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5194     }, {
5195         'note': 'Topic, should redirect to playlist?list=UU...',
5196         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5197         'info_dict': {
5198             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5199             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5200             'title': 'Uploads from Royalty Free Music - Topic',
5201             'uploader': 'Royalty Free Music - Topic',
5202             'tags': [],
5203             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5204             'channel': 'Royalty Free Music - Topic',
5205             'view_count': int,
5206             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5207             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5208             'modified_date': r're:\d{8}',
5209             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5210             'description': '',
5211         },
5212         'expected_warnings': [
5213             'The URL does not have a videos tab',
5214             r'[Uu]navailable videos (are|will be) hidden',
5215         ],
5216         'playlist_mincount': 101,
5217     }, {
5218         'note': 'Topic without a UU playlist',
5219         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5220         'info_dict': {
5221             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5222             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5223             'tags': [],
5224         },
5225         'expected_warnings': [
5226             'the playlist redirect gave error',
5227         ],
5228         'playlist_mincount': 9,
5229     }, {
5230         'note': 'Youtube music Album',
5231         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5232         'info_dict': {
5233             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5234             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5235             'tags': [],
5236             'view_count': int,
5237             'description': '',
5238             'availability': 'unlisted',
5239             'modified_date': r're:\d{8}',
5240         },
5241         'playlist_count': 50,
5242     }, {
5243         'note': 'unlisted single video playlist',
5244         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5245         'info_dict': {
5246             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5247             'uploader': 'colethedj',
5248             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5249             'title': 'yt-dlp unlisted playlist test',
5250             'availability': 'unlisted',
5251             'tags': [],
5252             'modified_date': '20220418',
5253             'channel': 'colethedj',
5254             'view_count': int,
5255             'description': '',
5256             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5257             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5258             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5259         },
5260         'playlist_count': 1,
5261     }, {
5262         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5263         'url': 'https://www.youtube.com/feed/recommended',
5264         'info_dict': {
5265             'id': 'recommended',
5266             'title': 'recommended',
5267             'tags': [],
5268         },
5269         'playlist_mincount': 50,
5270         'params': {
5271             'skip_download': True,
5272             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5273         },
5274     }, {
5275         'note': 'API Fallback: /videos tab, sorted by oldest first',
5276         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5277         'info_dict': {
5278             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5279             'title': 'Cody\'sLab - Videos',
5280             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5281             'uploader': 'Cody\'sLab',
5282             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5283             'channel': 'Cody\'sLab',
5284             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5285             'tags': [],
5286             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5287             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5288             'channel_follower_count': int
5289         },
5290         'playlist_mincount': 650,
5291         'params': {
5292             'skip_download': True,
5293             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5294         },
5295     }, {
5296         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5297         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5298         'info_dict': {
5299             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5300             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5301             'title': 'Uploads from Royalty Free Music - Topic',
5302             'uploader': 'Royalty Free Music - Topic',
5303             'modified_date': r're:\d{8}',
5304             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5305             'description': '',
5306             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5307             'tags': [],
5308             'channel': 'Royalty Free Music - Topic',
5309             'view_count': int,
5310             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5311         },
5312         'expected_warnings': [
5313             'does not have a videos tab',
5314             r'[Uu]navailable videos (are|will be) hidden',
5315         ],
5316         'playlist_mincount': 101,
5317         'params': {
5318             'skip_download': True,
5319             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5320         },
5321     }, {
5322         'note': 'non-standard redirect to regional channel',
5323         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5324         'only_matching': True
5325     }, {
5326         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5327         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5328         'info_dict': {
5329             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5330             'modified_date': '20220407',
5331             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5332             'tags': [],
5333             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5334             'uploader': 'pukkandan',
5335             'availability': 'unlisted',
5336             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5337             'channel': 'pukkandan',
5338             'description': 'Test for collaborative playlist',
5339             'title': 'yt-dlp test - collaborative playlist',
5340             'view_count': int,
5341             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5342         },
5343         'playlist_mincount': 2
5344     }]
5345
5346     @classmethod
5347     def suitable(cls, url):
5348         return False if YoutubeIE.suitable(url) else super().suitable(url)
5349
5350     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5351
5352     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5353     def _real_extract(self, url, smuggled_data):
5354         item_id = self._match_id(url)
5355         url = urllib.parse.urlunparse(
5356             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5357         compat_opts = self.get_param('compat_opts', [])
5358
5359         def get_mobj(url):
5360             mobj = self._URL_RE.match(url).groupdict()
5361             mobj.update((k, '') for k, v in mobj.items() if v is None)
5362             return mobj
5363
5364         mobj, redirect_warning = get_mobj(url), None
5365         # Youtube returns incomplete data if tabname is not lower case
5366         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5367         if is_channel:
5368             if smuggled_data.get('is_music_url'):
5369                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5370                     item_id = item_id[2:]
5371                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5372                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5373                     mdata = self._extract_tab_endpoint(
5374                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5375                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5376                                         get_all=False, expected_type=str)
5377                     if not murl:
5378                         raise ExtractorError('Failed to resolve album to playlist')
5379                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5380                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5381                     pre = f'https://www.youtube.com/channel/{item_id}'
5382
5383         original_tab_name = tab
5384         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5385             # Home URLs should redirect to /videos/
5386             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5387                                 'To download only the videos in the home page, add a "/featured" to the URL')
5388             tab = '/videos'
5389
5390         url = ''.join((pre, tab, post))
5391         mobj = get_mobj(url)
5392
5393         # Handle both video/playlist URLs
5394         qs = parse_qs(url)
5395         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5396
5397         if not video_id and mobj['not_channel'].startswith('watch'):
5398             if not playlist_id:
5399                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5400                 raise ExtractorError('Unable to recognize tab page')
5401             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5402             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5403             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5404             mobj = get_mobj(url)
5405
5406         if video_id and playlist_id:
5407             if self.get_param('noplaylist'):
5408                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5409                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5410                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5411             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5412
5413         data, ytcfg = self._extract_data(url, item_id)
5414
5415         # YouTube may provide a non-standard redirect to the regional channel
5416         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5417         redirect_url = traverse_obj(
5418             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5419         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5420             redirect_url = ''.join((
5421                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5422             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5423             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5424
5425         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5426         if tabs:
5427             selected_tab = self._extract_selected_tab(tabs)
5428             selected_tab_name = selected_tab.get('title', '').lower()
5429             if selected_tab_name == 'home':
5430                 selected_tab_name = 'featured'
5431             requested_tab_name = mobj['tab'][1:]
5432             if 'no-youtube-channel-redirect' not in compat_opts:
5433                 if requested_tab_name == 'live':  # Live tab should have redirected to the video
5434                     raise UserNotLive(video_id=mobj['id'])
5435                 if requested_tab_name not in ('', selected_tab_name):
5436                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5437                     if not original_tab_name:
5438                         if item_id[:2] == 'UC':
5439                             # Topic channels don't have /videos. Use the equivalent playlist instead
5440                             pl_id = f'UU{item_id[2:]}'
5441                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5442                             try:
5443                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5444                             except ExtractorError:
5445                                 redirect_warning += ' and the playlist redirect gave error'
5446                             else:
5447                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5448                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5449                         if selected_tab_name and selected_tab_name != requested_tab_name:
5450                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5451                     else:
5452                         raise ExtractorError(redirect_warning, expected=True)
5453
5454         if redirect_warning:
5455             self.to_screen(redirect_warning)
5456         self.write_debug(f'Final URL: {url}')
5457
5458         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5459         if 'no-youtube-unavailable-videos' not in compat_opts:
5460             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5461         self._extract_and_report_alerts(data, only_once=True)
5462         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5463         if tabs:
5464             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5465
5466         playlist = traverse_obj(
5467             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5468         if playlist:
5469             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5470
5471         video_id = traverse_obj(
5472             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5473         if video_id:
5474             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5475                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5476             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5477                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5478
5479         raise ExtractorError('Unable to recognize tab page')
5480
5481
5482 class YoutubePlaylistIE(InfoExtractor):
5483     IE_DESC = 'YouTube playlists'
5484     _VALID_URL = r'''(?x)(?:
5485                         (?:https?://)?
5486                         (?:\w+\.)?
5487                         (?:
5488                             (?:
5489                                 youtube(?:kids)?\.com|
5490                                 %(invidious)s
5491                             )
5492                             /.*?\?.*?\blist=
5493                         )?
5494                         (?P<id>%(playlist_id)s)
5495                      )''' % {
5496         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5497         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5498     }
5499     IE_NAME = 'youtube:playlist'
5500     _TESTS = [{
5501         'note': 'issue #673',
5502         'url': 'PLBB231211A4F62143',
5503         'info_dict': {
5504             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5505             'id': 'PLBB231211A4F62143',
5506             'uploader': 'Wickman',
5507             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5508             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5509             'view_count': int,
5510             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5511             'modified_date': r're:\d{8}',
5512             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5513             'channel': 'Wickman',
5514             'tags': [],
5515             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5516         },
5517         'playlist_mincount': 29,
5518     }, {
5519         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5520         'info_dict': {
5521             'title': 'YDL_safe_search',
5522             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5523         },
5524         'playlist_count': 2,
5525         'skip': 'This playlist is private',
5526     }, {
5527         'note': 'embedded',
5528         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5529         'playlist_count': 4,
5530         'info_dict': {
5531             'title': 'JODA15',
5532             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5533             'uploader': 'milan',
5534             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5535             'description': '',
5536             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5537             'tags': [],
5538             'modified_date': '20140919',
5539             'view_count': int,
5540             'channel': 'milan',
5541             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5542             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5543         },
5544         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5545     }, {
5546         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5547         'playlist_mincount': 455,
5548         'info_dict': {
5549             'title': '2018 Chinese New Singles (11/6 updated)',
5550             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5551             'uploader': 'LBK',
5552             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5553             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5554             'channel': 'LBK',
5555             'view_count': int,
5556             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5557             'tags': [],
5558             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5559             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5560             'modified_date': r're:\d{8}',
5561         },
5562         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5563     }, {
5564         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5565         'only_matching': True,
5566     }, {
5567         # music album playlist
5568         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5569         'only_matching': True,
5570     }]
5571
5572     @classmethod
5573     def suitable(cls, url):
5574         if YoutubeTabIE.suitable(url):
5575             return False
5576         from ..utils import parse_qs
5577         qs = parse_qs(url)
5578         if qs.get('v', [None])[0]:
5579             return False
5580         return super().suitable(url)
5581
5582     def _real_extract(self, url):
5583         playlist_id = self._match_id(url)
5584         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5585         url = update_url_query(
5586             'https://www.youtube.com/playlist',
5587             parse_qs(url) or {'list': playlist_id})
5588         if is_music_url:
5589             url = smuggle_url(url, {'is_music_url': True})
5590         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5591
5592
5593 class YoutubeYtBeIE(InfoExtractor):
5594     IE_DESC = 'youtu.be'
5595     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5596     _TESTS = [{
5597         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5598         'info_dict': {
5599             'id': 'yeWKywCrFtk',
5600             'ext': 'mp4',
5601             'title': 'Small Scale Baler and Braiding Rugs',
5602             'uploader': 'Backus-Page House Museum',
5603             'uploader_id': 'backuspagemuseum',
5604             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5605             'upload_date': '20161008',
5606             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5607             'categories': ['Nonprofits & Activism'],
5608             'tags': list,
5609             'like_count': int,
5610             'age_limit': 0,
5611             'playable_in_embed': True,
5612             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5613             'channel': 'Backus-Page House Museum',
5614             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5615             'live_status': 'not_live',
5616             'view_count': int,
5617             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5618             'availability': 'public',
5619             'duration': 59,
5620             'comment_count': int,
5621             'channel_follower_count': int
5622         },
5623         'params': {
5624             'noplaylist': True,
5625             'skip_download': True,
5626         },
5627     }, {
5628         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5629         'only_matching': True,
5630     }]
5631
5632     def _real_extract(self, url):
5633         mobj = self._match_valid_url(url)
5634         video_id = mobj.group('id')
5635         playlist_id = mobj.group('playlist_id')
5636         return self.url_result(
5637             update_url_query('https://www.youtube.com/watch', {
5638                 'v': video_id,
5639                 'list': playlist_id,
5640                 'feature': 'youtu.be',
5641             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5642
5643
5644 class YoutubeLivestreamEmbedIE(InfoExtractor):
5645     IE_DESC = 'YouTube livestream embeds'
5646     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5647     _TESTS = [{
5648         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5649         'only_matching': True,
5650     }]
5651
5652     def _real_extract(self, url):
5653         channel_id = self._match_id(url)
5654         return self.url_result(
5655             f'https://www.youtube.com/channel/{channel_id}/live',
5656             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5657
5658
5659 class YoutubeYtUserIE(InfoExtractor):
5660     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5661     IE_NAME = 'youtube:user'
5662     _VALID_URL = r'ytuser:(?P<id>.+)'
5663     _TESTS = [{
5664         'url': 'ytuser:phihag',
5665         'only_matching': True,
5666     }]
5667
5668     def _real_extract(self, url):
5669         user_id = self._match_id(url)
5670         return self.url_result(
5671             'https://www.youtube.com/user/%s/videos' % user_id,
5672             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5673
5674
5675 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5676     IE_NAME = 'youtube:favorites'
5677     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5678     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5679     _LOGIN_REQUIRED = True
5680     _TESTS = [{
5681         'url': ':ytfav',
5682         'only_matching': True,
5683     }, {
5684         'url': ':ytfavorites',
5685         'only_matching': True,
5686     }]
5687
5688     def _real_extract(self, url):
5689         return self.url_result(
5690             'https://www.youtube.com/playlist?list=LL',
5691             ie=YoutubeTabIE.ie_key())
5692
5693
5694 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5695     IE_NAME = 'youtube:notif'
5696     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5697     _VALID_URL = r':ytnotif(?:ication)?s?'
5698     _LOGIN_REQUIRED = True
5699     _TESTS = [{
5700         'url': ':ytnotif',
5701         'only_matching': True,
5702     }, {
5703         'url': ':ytnotifications',
5704         'only_matching': True,
5705     }]
5706
5707     def _extract_notification_menu(self, response, continuation_list):
5708         notification_list = traverse_obj(
5709             response,
5710             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5711             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5712             expected_type=list) or []
5713         continuation_list[0] = None
5714         for item in notification_list:
5715             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5716             if entry:
5717                 yield entry
5718             continuation = item.get('continuationItemRenderer')
5719             if continuation:
5720                 continuation_list[0] = continuation
5721
5722     def _extract_notification_renderer(self, notification):
5723         video_id = traverse_obj(
5724             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5725         url = f'https://www.youtube.com/watch?v={video_id}'
5726         channel_id = None
5727         if not video_id:
5728             browse_ep = traverse_obj(
5729                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5730             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5731             post_id = self._search_regex(
5732                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5733                 'post id', default=None)
5734             if not channel_id or not post_id:
5735                 return
5736             # The direct /post url redirects to this in the browser
5737             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5738
5739         channel = traverse_obj(
5740             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5741             expected_type=str)
5742         notification_title = self._get_text(notification, 'shortMessage')
5743         if notification_title:
5744             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
5745         # TODO: handle recommended videos
5746         title = self._search_regex(
5747             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
5748             'video title', default=None)
5749         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5750                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5751                        else None)
5752         return {
5753             '_type': 'url',
5754             'url': url,
5755             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5756             'video_id': video_id,
5757             'title': title,
5758             'channel_id': channel_id,
5759             'channel': channel,
5760             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5761             'upload_date': upload_date,
5762         }
5763
5764     def _notification_menu_entries(self, ytcfg):
5765         continuation_list = [None]
5766         response = None
5767         for page in itertools.count(1):
5768             ctoken = traverse_obj(
5769                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5770             response = self._extract_response(
5771                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5772                 ep='notification/get_notification_menu', check_get_keys='actions',
5773                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5774             yield from self._extract_notification_menu(response, continuation_list)
5775             if not continuation_list[0]:
5776                 break
5777
5778     def _real_extract(self, url):
5779         display_id = 'notifications'
5780         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5781         self._report_playlist_authcheck(ytcfg)
5782         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5783
5784
5785 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5786     IE_DESC = 'YouTube search'
5787     IE_NAME = 'youtube:search'
5788     _SEARCH_KEY = 'ytsearch'
5789     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5790     _TESTS = [{
5791         'url': 'ytsearch5:youtube-dl test video',
5792         'playlist_count': 5,
5793         'info_dict': {
5794             'id': 'youtube-dl test video',
5795             'title': 'youtube-dl test video',
5796         }
5797     }]
5798
5799
5800 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5801     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5802     _SEARCH_KEY = 'ytsearchdate'
5803     IE_DESC = 'YouTube search, newest videos first'
5804     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5805     _TESTS = [{
5806         'url': 'ytsearchdate5:youtube-dl test video',
5807         'playlist_count': 5,
5808         'info_dict': {
5809             'id': 'youtube-dl test video',
5810             'title': 'youtube-dl test video',
5811         }
5812     }]
5813
5814
5815 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5816     IE_DESC = 'YouTube search URLs with sorting and filter support'
5817     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5818     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5819     _TESTS = [{
5820         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5821         'playlist_mincount': 5,
5822         'info_dict': {
5823             'id': 'youtube-dl test video',
5824             'title': 'youtube-dl test video',
5825         }
5826     }, {
5827         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5828         'playlist_mincount': 5,
5829         'info_dict': {
5830             'id': 'python',
5831             'title': 'python',
5832         }
5833     }, {
5834         'url': 'https://www.youtube.com/results?search_query=%23cats',
5835         'playlist_mincount': 1,
5836         'info_dict': {
5837             'id': '#cats',
5838             'title': '#cats',
5839             # The test suite does not have support for nested playlists
5840             # 'entries': [{
5841             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5842             #     'title': '#cats',
5843             # }],
5844         },
5845     }, {
5846         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5847         'only_matching': True,
5848     }]
5849
5850     def _real_extract(self, url):
5851         qs = parse_qs(url)
5852         query = (qs.get('search_query') or qs.get('q'))[0]
5853         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5854
5855
5856 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5857     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5858     IE_NAME = 'youtube:music:search_url'
5859     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5860     _TESTS = [{
5861         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5862         'playlist_count': 16,
5863         'info_dict': {
5864             'id': 'royalty free music',
5865             'title': 'royalty free music',
5866         }
5867     }, {
5868         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5869         'playlist_mincount': 30,
5870         'info_dict': {
5871             'id': 'royalty free music - songs',
5872             'title': 'royalty free music - songs',
5873         },
5874         'params': {'extract_flat': 'in_playlist'}
5875     }, {
5876         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5877         'playlist_mincount': 30,
5878         'info_dict': {
5879             'id': 'royalty free music - community playlists',
5880             'title': 'royalty free music - community playlists',
5881         },
5882         'params': {'extract_flat': 'in_playlist'}
5883     }]
5884
5885     _SECTIONS = {
5886         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5887         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5888         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5889         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5890         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5891         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5892     }
5893
5894     def _real_extract(self, url):
5895         qs = parse_qs(url)
5896         query = (qs.get('search_query') or qs.get('q'))[0]
5897         params = qs.get('sp', (None,))[0]
5898         if params:
5899             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5900         else:
5901             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
5902             params = self._SECTIONS.get(section)
5903             if not params:
5904                 section = None
5905         title = join_nonempty(query, section, delim=' - ')
5906         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5907
5908
5909 class YoutubeFeedsInfoExtractor(InfoExtractor):
5910     """
5911     Base class for feed extractors
5912     Subclasses must re-define the _FEED_NAME property.
5913     """
5914     _LOGIN_REQUIRED = True
5915     _FEED_NAME = 'feeds'
5916
5917     def _real_initialize(self):
5918         YoutubeBaseInfoExtractor._check_login_required(self)
5919
5920     @classproperty
5921     def IE_NAME(self):
5922         return f'youtube:{self._FEED_NAME}'
5923
5924     def _real_extract(self, url):
5925         return self.url_result(
5926             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5927
5928
5929 class YoutubeWatchLaterIE(InfoExtractor):
5930     IE_NAME = 'youtube:watchlater'
5931     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5932     _VALID_URL = r':ytwatchlater'
5933     _TESTS = [{
5934         'url': ':ytwatchlater',
5935         'only_matching': True,
5936     }]
5937
5938     def _real_extract(self, url):
5939         return self.url_result(
5940             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5941
5942
5943 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5944     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5945     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5946     _FEED_NAME = 'recommended'
5947     _LOGIN_REQUIRED = False
5948     _TESTS = [{
5949         'url': ':ytrec',
5950         'only_matching': True,
5951     }, {
5952         'url': ':ytrecommended',
5953         'only_matching': True,
5954     }, {
5955         'url': 'https://youtube.com',
5956         'only_matching': True,
5957     }]
5958
5959
5960 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5961     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5962     _VALID_URL = r':ytsub(?:scription)?s?'
5963     _FEED_NAME = 'subscriptions'
5964     _TESTS = [{
5965         'url': ':ytsubs',
5966         'only_matching': True,
5967     }, {
5968         'url': ':ytsubscriptions',
5969         'only_matching': True,
5970     }]
5971
5972
5973 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5974     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5975     _VALID_URL = r':ythis(?:tory)?'
5976     _FEED_NAME = 'history'
5977     _TESTS = [{
5978         'url': ':ythistory',
5979         'only_matching': True,
5980     }]
5981
5982
5983 class YoutubeStoriesIE(InfoExtractor):
5984     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
5985     IE_NAME = 'youtube:stories'
5986     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
5987     _TESTS = [{
5988         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
5989         'only_matching': True,
5990     }]
5991
5992     def _real_extract(self, url):
5993         playlist_id = f'RLTD{self._match_id(url)}'
5994         return self.url_result(
5995             f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1',
5996             ie=YoutubeTabIE, video_id=playlist_id)
5997
5998
5999 class YoutubeTruncatedURLIE(InfoExtractor):
6000     IE_NAME = 'youtube:truncated_url'
6001     IE_DESC = False  # Do not list
6002     _VALID_URL = r'''(?x)
6003         (?:https?://)?
6004         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6005         (?:watch\?(?:
6006             feature=[a-z_]+|
6007             annotation_id=annotation_[^&]+|
6008             x-yt-cl=[0-9]+|
6009             hl=[^&]*|
6010             t=[0-9]+
6011         )?
6012         |
6013             attribution_link\?a=[^&]+
6014         )
6015         $
6016     '''
6017
6018     _TESTS = [{
6019         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6020         'only_matching': True,
6021     }, {
6022         'url': 'https://www.youtube.com/watch?',
6023         'only_matching': True,
6024     }, {
6025         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6026         'only_matching': True,
6027     }, {
6028         'url': 'https://www.youtube.com/watch?feature=foo',
6029         'only_matching': True,
6030     }, {
6031         'url': 'https://www.youtube.com/watch?hl=en-GB',
6032         'only_matching': True,
6033     }, {
6034         'url': 'https://www.youtube.com/watch?t=2372',
6035         'only_matching': True,
6036     }]
6037
6038     def _real_extract(self, url):
6039         raise ExtractorError(
6040             'Did you forget to quote the URL? Remember that & is a meta '
6041             'character in most shells, so you want to put the URL in quotes, '
6042             'like  youtube-dl '
6043             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6044             ' or simply  youtube-dl BaW_jenozKc  .',
6045             expected=True)
6046
6047
6048 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6049     IE_NAME = 'youtube:clip'
6050     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6051     _TESTS = [{
6052         # FIXME: Other metadata should be extracted from the clip, not from the base video
6053         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6054         'info_dict': {
6055             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6056             'ext': 'mp4',
6057             'section_start': 29.0,
6058             'section_end': 39.7,
6059             'duration': 10.7,
6060             'age_limit': 0,
6061             'availability': 'public',
6062             'categories': ['Gaming'],
6063             'channel': 'Scott The Woz',
6064             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6065             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6066             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6067             'like_count': int,
6068             'playable_in_embed': True,
6069             'tags': 'count:17',
6070             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6071             'title': 'Mobile Games on Console - Scott The Woz',
6072             'upload_date': '20210920',
6073             'uploader': 'Scott The Woz',
6074             'uploader_id': 'scottthewoz',
6075             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6076             'view_count': int,
6077             'live_status': 'not_live',
6078             'channel_follower_count': int
6079         }
6080     }]
6081
6082     def _real_extract(self, url):
6083         clip_id = self._match_id(url)
6084         _, data = self._extract_webpage(url, clip_id)
6085
6086         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6087         if not video_id:
6088             raise ExtractorError('Unable to find video ID')
6089
6090         clip_data = traverse_obj(data, (
6091             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6092             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6093             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6094             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6095
6096         return {
6097             '_type': 'url_transparent',
6098             'url': f'https://www.youtube.com/watch?v={video_id}',
6099             'ie_key': YoutubeIE.ie_key(),
6100             'id': clip_id,
6101             'section_start': int(clip_data['startTimeMs']) / 1000,
6102             'section_end': int(clip_data['endTimeMs']) / 1000,
6103         }
6104
6105
6106 class YoutubeTruncatedIDIE(InfoExtractor):
6107     IE_NAME = 'youtube:truncated_id'
6108     IE_DESC = False  # Do not list
6109     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6110
6111     _TESTS = [{
6112         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6113         'only_matching': True,
6114     }]
6115
6116     def _real_extract(self, url):
6117         video_id = self._match_id(url)
6118         raise ExtractorError(
6119             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6120             expected=True)