yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import copy
   4 import datetime
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import threading
  14 import time
  15 import traceback
  16 import urllib.error
  17 import urllib.parse
  18
  19 from .common import InfoExtractor, SearchInfoExtractor
  20 from .openload import PhantomJSwrapper
  21 from ..compat import functools
  22 from ..jsinterp import JSInterpreter
  23 from ..utils import (
  24     NO_DEFAULT,
  25     ExtractorError,
  26     UserNotLive,
  27     bug_reports_message,
  28     classproperty,
  29     clean_html,
  30     datetime_from_str,
  31     dict_get,
  32     float_or_none,
  33     format_field,
  34     get_first,
  35     int_or_none,
  36     is_html,
  37     join_nonempty,
  38     js_to_json,
  39     mimetype2ext,
  40     network_exceptions,
  41     orderedSet,
  42     parse_codecs,
  43     parse_count,
  44     parse_duration,
  45     parse_iso8601,
  46     parse_qs,
  47     qualities,
  48     remove_start,
  49     smuggle_url,
  50     str_or_none,
  51     str_to_int,
  52     strftime_or_none,
  53     traverse_obj,
  54     try_get,
  55     unescapeHTML,
  56     unified_strdate,
  57     unified_timestamp,
  58     unsmuggle_url,
  59     update_url_query,
  60     url_or_none,
  61     urljoin,
  62     variadic,
  63 )
  64
  65 # any clients starting with _ cannot be explicitly requested by the user
  66 INNERTUBE_CLIENTS = {
  67     'web': {
  68         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  69         'INNERTUBE_CONTEXT': {
  70             'client': {
  71                 'clientName': 'WEB',
  72                 'clientVersion': '2.20220801.00.00',
  73             }
  74         },
  75         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  76     },
  77     'web_embedded': {
  78         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  79         'INNERTUBE_CONTEXT': {
  80             'client': {
  81                 'clientName': 'WEB_EMBEDDED_PLAYER',
  82                 'clientVersion': '1.20220731.00.00',
  83             },
  84         },
  85         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  86     },
  87     'web_music': {
  88         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  89         'INNERTUBE_HOST': 'music.youtube.com',
  90         'INNERTUBE_CONTEXT': {
  91             'client': {
  92                 'clientName': 'WEB_REMIX',
  93                 'clientVersion': '1.20220727.01.00',
  94             }
  95         },
  96         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
  97     },
  98     'web_creator': {
  99         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 100         'INNERTUBE_CONTEXT': {
 101             'client': {
 102                 'clientName': 'WEB_CREATOR',
 103                 'clientVersion': '1.20220726.00.00',
 104             }
 105         },
 106         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 107     },
 108     'android': {
 109         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 110         'INNERTUBE_CONTEXT': {
 111             'client': {
 112                 'clientName': 'ANDROID',
 113                 'clientVersion': '17.31.35',
 114                 'androidSdkVersion': 30,
 115                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 116             }
 117         },
 118         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 119         'REQUIRE_JS_PLAYER': False
 120     },
 121     'android_embedded': {
 122         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 123         'INNERTUBE_CONTEXT': {
 124             'client': {
 125                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 126                 'clientVersion': '17.31.35',
 127                 'androidSdkVersion': 30,
 128                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 129             },
 130         },
 131         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 132         'REQUIRE_JS_PLAYER': False
 133     },
 134     'android_music': {
 135         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 136         'INNERTUBE_CONTEXT': {
 137             'client': {
 138                 'clientName': 'ANDROID_MUSIC',
 139                 'clientVersion': '5.16.51',
 140                 'androidSdkVersion': 30,
 141                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 142             }
 143         },
 144         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 145         'REQUIRE_JS_PLAYER': False
 146     },
 147     'android_creator': {
 148         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 149         'INNERTUBE_CONTEXT': {
 150             'client': {
 151                 'clientName': 'ANDROID_CREATOR',
 152                 'clientVersion': '22.30.100',
 153                 'androidSdkVersion': 30,
 154                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 155             },
 156         },
 157         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 158         'REQUIRE_JS_PLAYER': False
 159     },
 160     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 161     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 162     'ios': {
 163         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 164         'INNERTUBE_CONTEXT': {
 165             'client': {
 166                 'clientName': 'IOS',
 167                 'clientVersion': '17.33.2',
 168                 'deviceModel': 'iPhone14,3',
 169                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 170             }
 171         },
 172         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 173         'REQUIRE_JS_PLAYER': False
 174     },
 175     'ios_embedded': {
 176         'INNERTUBE_CONTEXT': {
 177             'client': {
 178                 'clientName': 'IOS_MESSAGES_EXTENSION',
 179                 'clientVersion': '17.33.2',
 180                 'deviceModel': 'iPhone14,3',
 181                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 182             },
 183         },
 184         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 185         'REQUIRE_JS_PLAYER': False
 186     },
 187     'ios_music': {
 188         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 189         'INNERTUBE_CONTEXT': {
 190             'client': {
 191                 'clientName': 'IOS_MUSIC',
 192                 'clientVersion': '5.21',
 193                 'deviceModel': 'iPhone14,3',
 194                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 195             },
 196         },
 197         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 198         'REQUIRE_JS_PLAYER': False
 199     },
 200     'ios_creator': {
 201         'INNERTUBE_CONTEXT': {
 202             'client': {
 203                 'clientName': 'IOS_CREATOR',
 204                 'clientVersion': '22.33.101',
 205                 'deviceModel': 'iPhone14,3',
 206                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 207             },
 208         },
 209         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 210         'REQUIRE_JS_PLAYER': False
 211     },
 212     # mweb has 'ultralow' formats
 213     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 214     'mweb': {
 215         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 216         'INNERTUBE_CONTEXT': {
 217             'client': {
 218                 'clientName': 'MWEB',
 219                 'clientVersion': '2.20220801.00.00',
 220             }
 221         },
 222         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 223     },
 224     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 225     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 226     'tv_embedded': {
 227         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 228         'INNERTUBE_CONTEXT': {
 229             'client': {
 230                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 231                 'clientVersion': '2.0',
 232             },
 233         },
 234         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 235     },
 236 }
 237
 238
 239 def _split_innertube_client(client_name):
 240     variant, *base = client_name.rsplit('.', 1)
 241     if base:
 242         return variant, base[0], variant
 243     base, *variant = client_name.split('_', 1)
 244     return client_name, base, variant[0] if variant else None
 245
 246
 247 def build_innertube_clients():
 248     THIRD_PARTY = {
 249         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 250     }
 251     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 252     priority = qualities(BASE_CLIENTS[::-1])
 253
 254     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 255         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 256         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 257         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 258         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 259
 260         _, base_client, variant = _split_innertube_client(client)
 261         ytcfg['priority'] = 10 * priority(base_client)
 262
 263         if not variant:
 264             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 265             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 266             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 267             embedscreen['priority'] -= 3
 268         elif variant == 'embedded':
 269             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 270             ytcfg['priority'] -= 2
 271         else:
 272             ytcfg['priority'] -= 3
 273
 274
 275 build_innertube_clients()
 276
 277
 278 class YoutubeBaseInfoExtractor(InfoExtractor):
 279     """Provide base functions for Youtube extractors"""
 280
 281     _RESERVED_NAMES = (
 282         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 283         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 284         r'browse|oembed|get_video_info|iframe_api|s/player|'
 285         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 286
 287     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 288
 289     # _NETRC_MACHINE = 'youtube'
 290
 291     # If True it will raise an error if no login info is provided
 292     _LOGIN_REQUIRED = False
 293
 294     _INVIDIOUS_SITES = (
 295         # invidious-redirect websites
 296         r'(?:www\.)?redirect\.invidious\.io',
 297         r'(?:(?:www|dev)\.)?invidio\.us',
 298         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 299         r'(?:www\.)?invidious\.pussthecat\.org',
 300         r'(?:www\.)?invidious\.zee\.li',
 301         r'(?:www\.)?invidious\.ethibox\.fr',
 302         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 303         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 304         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 305         # youtube-dl invidious instances list
 306         r'(?:(?:www|no)\.)?invidiou\.sh',
 307         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 308         r'(?:www\.)?invidious\.kabi\.tk',
 309         r'(?:www\.)?invidious\.mastodon\.host',
 310         r'(?:www\.)?invidious\.zapashcanon\.fr',
 311         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 312         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 313         r'(?:www\.)?invidious\.himiko\.cloud',
 314         r'(?:www\.)?invidious\.reallyancient\.tech',
 315         r'(?:www\.)?invidious\.tube',
 316         r'(?:www\.)?invidiou\.site',
 317         r'(?:www\.)?invidious\.site',
 318         r'(?:www\.)?invidious\.xyz',
 319         r'(?:www\.)?invidious\.nixnet\.xyz',
 320         r'(?:www\.)?invidious\.048596\.xyz',
 321         r'(?:www\.)?invidious\.drycat\.fr',
 322         r'(?:www\.)?inv\.skyn3t\.in',
 323         r'(?:www\.)?tube\.poal\.co',
 324         r'(?:www\.)?tube\.connect\.cafe',
 325         r'(?:www\.)?vid\.wxzm\.sx',
 326         r'(?:www\.)?vid\.mint\.lgbt',
 327         r'(?:www\.)?vid\.puffyan\.us',
 328         r'(?:www\.)?yewtu\.be',
 329         r'(?:www\.)?yt\.elukerio\.org',
 330         r'(?:www\.)?yt\.lelux\.fi',
 331         r'(?:www\.)?invidious\.ggc-project\.de',
 332         r'(?:www\.)?yt\.maisputain\.ovh',
 333         r'(?:www\.)?ytprivate\.com',
 334         r'(?:www\.)?invidious\.13ad\.de',
 335         r'(?:www\.)?invidious\.toot\.koeln',
 336         r'(?:www\.)?invidious\.fdn\.fr',
 337         r'(?:www\.)?watch\.nettohikari\.com',
 338         r'(?:www\.)?invidious\.namazso\.eu',
 339         r'(?:www\.)?invidious\.silkky\.cloud',
 340         r'(?:www\.)?invidious\.exonip\.de',
 341         r'(?:www\.)?invidious\.riverside\.rocks',
 342         r'(?:www\.)?invidious\.blamefran\.net',
 343         r'(?:www\.)?invidious\.moomoo\.de',
 344         r'(?:www\.)?ytb\.trom\.tf',
 345         r'(?:www\.)?yt\.cyberhost\.uk',
 346         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 347         r'(?:www\.)?qklhadlycap4cnod\.onion',
 348         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 349         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 350         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 351         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 352         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 353         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 354         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 355         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 356         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 357         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 358         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 359         r'(?:www\.)?piped\.kavin\.rocks',
 360         r'(?:www\.)?piped\.silkky\.cloud',
 361         r'(?:www\.)?piped\.tokhmi\.xyz',
 362         r'(?:www\.)?piped\.moomoo\.me',
 363         r'(?:www\.)?il\.ax',
 364         r'(?:www\.)?piped\.syncpundit\.com',
 365         r'(?:www\.)?piped\.mha\.fi',
 366         r'(?:www\.)?piped\.mint\.lgbt',
 367         r'(?:www\.)?piped\.privacy\.com\.de',
 368     )
 369
 370     def _initialize_consent(self):
 371         cookies = self._get_cookies('https://www.youtube.com/')
 372         if cookies.get('__Secure-3PSID'):
 373             return
 374         consent_id = None
 375         consent = cookies.get('CONSENT')
 376         if consent:
 377             if 'YES' in consent.value:
 378                 return
 379             consent_id = self._search_regex(
 380                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 381         if not consent_id:
 382             consent_id = random.randint(100, 999)
 383         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 384
 385     def _initialize_pref(self):
 386         cookies = self._get_cookies('https://www.youtube.com/')
 387         pref_cookie = cookies.get('PREF')
 388         pref = {}
 389         if pref_cookie:
 390             try:
 391                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 392             except ValueError:
 393                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 394         pref.update({'hl': 'en', 'tz': 'UTC'})
 395         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 396
 397     def _real_initialize(self):
 398         self._initialize_pref()
 399         self._initialize_consent()
 400         self._check_login_required()
 401
 402     def _check_login_required(self):
 403         if self._LOGIN_REQUIRED and not self._cookies_passed:
 404             self.raise_login_required('Login details are needed to download this content', method='cookies')
 405
 406     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 407     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 408
 409     def _get_default_ytcfg(self, client='web'):
 410         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 411
 412     def _get_innertube_host(self, client='web'):
 413         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 414
 415     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 416         # try_get but with fallback to default ytcfg client values when present
 417         _func = lambda y: try_get(y, getter, expected_type)
 418         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 419
 420     def _extract_client_name(self, ytcfg, default_client='web'):
 421         return self._ytcfg_get_safe(
 422             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 423                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 424
 425     def _extract_client_version(self, ytcfg, default_client='web'):
 426         return self._ytcfg_get_safe(
 427             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 428                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 429
 430     def _select_api_hostname(self, req_api_hostname, default_client=None):
 431         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 432                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 433
 434     def _extract_api_key(self, ytcfg=None, default_client='web'):
 435         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 436
 437     def _extract_context(self, ytcfg=None, default_client='web'):
 438         context = get_first(
 439             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 440         # Enforce language and tz for extraction
 441         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 442         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 443         return context
 444
 445     _SAPISID = None
 446
 447     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 448         time_now = round(time.time())
 449         if self._SAPISID is None:
 450             yt_cookies = self._get_cookies('https://www.youtube.com')
 451             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 452             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 453             sapisid_cookie = dict_get(
 454                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 455             if sapisid_cookie and sapisid_cookie.value:
 456                 self._SAPISID = sapisid_cookie.value
 457                 self.write_debug('Extracted SAPISID cookie')
 458                 # SAPISID cookie is required if not already present
 459                 if not yt_cookies.get('SAPISID'):
 460                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 461                     self._set_cookie(
 462                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 463             else:
 464                 self._SAPISID = False
 465         if not self._SAPISID:
 466             return None
 467         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 468         sapisidhash = hashlib.sha1(
 469             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 470         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 471
 472     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 473                   note='Downloading API JSON', errnote='Unable to download API page',
 474                   context=None, api_key=None, api_hostname=None, default_client='web'):
 475
 476         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 477         data.update(query)
 478         real_headers = self.generate_api_headers(default_client=default_client)
 479         real_headers.update({'content-type': 'application/json'})
 480         if headers:
 481             real_headers.update(headers)
 482         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 483                    or api_key or self._extract_api_key(default_client=default_client))
 484         return self._download_json(
 485             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 486             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 487             data=json.dumps(data).encode('utf8'), headers=real_headers,
 488             query={'key': api_key, 'prettyPrint': 'false'})
 489
 490     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 491         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 492
 493     @staticmethod
 494     def _extract_session_index(*data):
 495         """
 496         Index of current account in account list.
 497         See: https://github.com/yt-dlp/yt-dlp/pull/519
 498         """
 499         for ytcfg in data:
 500             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 501             if session_index is not None:
 502                 return session_index
 503
 504     # Deprecated?
 505     def _extract_identity_token(self, ytcfg=None, webpage=None):
 506         if ytcfg:
 507             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 508             if token:
 509                 return token
 510         if webpage:
 511             return self._search_regex(
 512                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 513                 'identity token', default=None, fatal=False)
 514
 515     @staticmethod
 516     def _extract_account_syncid(*args):
 517         """
 518         Extract syncId required to download private playlists of secondary channels
 519         @params response and/or ytcfg
 520         """
 521         for data in args:
 522             # ytcfg includes channel_syncid if on secondary channel
 523             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 524             if delegated_sid:
 525                 return delegated_sid
 526             sync_ids = (try_get(
 527                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 528                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 529             if len(sync_ids) >= 2 and sync_ids[1]:
 530                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 531                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 532                 return sync_ids[0]
 533
 534     @staticmethod
 535     def _extract_visitor_data(*args):
 536         """
 537         Extracts visitorData from an API response or ytcfg
 538         Appears to be used to track session state
 539         """
 540         return get_first(
 541             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 542             expected_type=str)
 543
 544     @functools.cached_property
 545     def is_authenticated(self):
 546         return bool(self._generate_sapisidhash_header())
 547
 548     def extract_ytcfg(self, video_id, webpage):
 549         if not webpage:
 550             return {}
 551         return self._parse_json(
 552             self._search_regex(
 553                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 554                 default='{}'), video_id, fatal=False) or {}
 555
 556     def generate_api_headers(
 557             self, *, ytcfg=None, account_syncid=None, session_index=None,
 558             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 559
 560         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 561         headers = {
 562             'X-YouTube-Client-Name': str(
 563                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 564             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 565             'Origin': origin,
 566             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 567             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 568             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 569             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 570         }
 571         if session_index is None:
 572             session_index = self._extract_session_index(ytcfg)
 573         if account_syncid or session_index is not None:
 574             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 575
 576         auth = self._generate_sapisidhash_header(origin)
 577         if auth is not None:
 578             headers['Authorization'] = auth
 579             headers['X-Origin'] = origin
 580         return {h: v for h, v in headers.items() if v is not None}
 581
 582     def _download_ytcfg(self, client, video_id):
 583         url = {
 584             'web': 'https://www.youtube.com',
 585             'web_music': 'https://music.youtube.com',
 586             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 587         }.get(client)
 588         if not url:
 589             return {}
 590         webpage = self._download_webpage(
 591             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 592         return self.extract_ytcfg(video_id, webpage) or {}
 593
 594     @staticmethod
 595     def _build_api_continuation_query(continuation, ctp=None):
 596         query = {
 597             'continuation': continuation
 598         }
 599         # TODO: Inconsistency with clickTrackingParams.
 600         # Currently we have a fixed ctp contained within context (from ytcfg)
 601         # and a ctp in root query for continuation.
 602         if ctp:
 603             query['clickTracking'] = {'clickTrackingParams': ctp}
 604         return query
 605
 606     @classmethod
 607     def _extract_next_continuation_data(cls, renderer):
 608         next_continuation = try_get(
 609             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 610                        lambda x: x['continuation']['reloadContinuationData']), dict)
 611         if not next_continuation:
 612             return
 613         continuation = next_continuation.get('continuation')
 614         if not continuation:
 615             return
 616         ctp = next_continuation.get('clickTrackingParams')
 617         return cls._build_api_continuation_query(continuation, ctp)
 618
 619     @classmethod
 620     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 621         if isinstance(continuation_ep, dict):
 622             continuation = try_get(
 623                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 624             if not continuation:
 625                 return
 626             ctp = continuation_ep.get('clickTrackingParams')
 627             return cls._build_api_continuation_query(continuation, ctp)
 628
 629     @classmethod
 630     def _extract_continuation(cls, renderer):
 631         next_continuation = cls._extract_next_continuation_data(renderer)
 632         if next_continuation:
 633             return next_continuation
 634
 635         contents = []
 636         for key in ('contents', 'items'):
 637             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 638
 639         for content in contents:
 640             if not isinstance(content, dict):
 641                 continue
 642             continuation_ep = try_get(
 643                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 644                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 645                 dict)
 646             continuation = cls._extract_continuation_ep_data(continuation_ep)
 647             if continuation:
 648                 return continuation
 649
 650     @classmethod
 651     def _extract_alerts(cls, data):
 652         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 653             if not isinstance(alert_dict, dict):
 654                 continue
 655             for alert in alert_dict.values():
 656                 alert_type = alert.get('type')
 657                 if not alert_type:
 658                     continue
 659                 message = cls._get_text(alert, 'text')
 660                 if message:
 661                     yield alert_type, message
 662
 663     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 664         errors = []
 665         warnings = []
 666         for alert_type, alert_message in alerts:
 667             if alert_type.lower() == 'error' and fatal:
 668                 errors.append([alert_type, alert_message])
 669             else:
 670                 warnings.append([alert_type, alert_message])
 671
 672         for alert_type, alert_message in (warnings + errors[:-1]):
 673             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 674         if errors:
 675             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 676
 677     def _extract_and_report_alerts(self, data, *args, **kwargs):
 678         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 679
 680     def _extract_badges(self, renderer: dict):
 681         badges = set()
 682         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 683             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], str)
 684             if label:
 685                 badges.add(label.lower())
 686         return badges
 687
 688     @staticmethod
 689     def _get_text(data, *path_list, max_runs=None):
 690         for path in path_list or [None]:
 691             if path is None:
 692                 obj = [data]
 693             else:
 694                 obj = traverse_obj(data, path, default=[])
 695                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 696                     obj = [obj]
 697             for item in obj:
 698                 text = try_get(item, lambda x: x['simpleText'], str)
 699                 if text:
 700                     return text
 701                 runs = try_get(item, lambda x: x['runs'], list) or []
 702                 if not runs and isinstance(item, list):
 703                     runs = item
 704
 705                 runs = runs[:min(len(runs), max_runs or len(runs))]
 706                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 707                 if text:
 708                     return text
 709
 710     def _get_count(self, data, *path_list):
 711         count_text = self._get_text(data, *path_list) or ''
 712         count = parse_count(count_text)
 713         if count is None:
 714             count = str_to_int(
 715                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 716         return count
 717
 718     @staticmethod
 719     def _extract_thumbnails(data, *path_list):
 720         """
 721         Extract thumbnails from thumbnails dict
 722         @param path_list: path list to level that contains 'thumbnails' key
 723         """
 724         thumbnails = []
 725         for path in path_list or [()]:
 726             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 727                 thumbnail_url = url_or_none(thumbnail.get('url'))
 728                 if not thumbnail_url:
 729                     continue
 730                 # Sometimes youtube gives a wrong thumbnail URL. See:
 731                 # https://github.com/yt-dlp/yt-dlp/issues/233
 732                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 733                 if 'maxresdefault' in thumbnail_url:
 734                     thumbnail_url = thumbnail_url.split('?')[0]
 735                 thumbnails.append({
 736                     'url': thumbnail_url,
 737                     'height': int_or_none(thumbnail.get('height')),
 738                     'width': int_or_none(thumbnail.get('width')),
 739                 })
 740         return thumbnails
 741
 742     @staticmethod
 743     def extract_relative_time(relative_time_text):
 744         """
 745         Extracts a relative time from string and converts to dt object
 746         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 747         """
 748         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 749         if mobj:
 750             start = mobj.group('start')
 751             if start:
 752                 return datetime_from_str(start)
 753             try:
 754                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 755             except ValueError:
 756                 return None
 757
 758     def _extract_time_text(self, renderer, *path_list):
 759         """@returns (timestamp, time_text)"""
 760         text = self._get_text(renderer, *path_list) or ''
 761         dt = self.extract_relative_time(text)
 762         timestamp = None
 763         if isinstance(dt, datetime.datetime):
 764             timestamp = calendar.timegm(dt.timetuple())
 765
 766         if timestamp is None:
 767             timestamp = (
 768                 unified_timestamp(text) or unified_timestamp(
 769                     self._search_regex(
 770                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 771                         text.lower(), 'time text', default=None)))
 772
 773         if text and timestamp is None:
 774             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 775         return timestamp, text
 776
 777     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 778                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 779                           default_client='web'):
 780         for retry in self.RetryManager():
 781             try:
 782                 response = self._call_api(
 783                     ep=ep, fatal=True, headers=headers,
 784                     video_id=item_id, query=query, note=note,
 785                     context=self._extract_context(ytcfg, default_client),
 786                     api_key=self._extract_api_key(ytcfg, default_client),
 787                     api_hostname=api_hostname, default_client=default_client)
 788             except ExtractorError as e:
 789                 if not isinstance(e.cause, network_exceptions):
 790                     return self._error_or_warning(e, fatal=fatal)
 791                 elif not isinstance(e.cause, urllib.error.HTTPError):
 792                     retry.error = e
 793                     continue
 794
 795                 first_bytes = e.cause.read(512)
 796                 if not is_html(first_bytes):
 797                     yt_error = try_get(
 798                         self._parse_json(
 799                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 800                         lambda x: x['error']['message'], str)
 801                     if yt_error:
 802                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 803                 # Downloading page may result in intermittent 5xx HTTP error
 804                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 805                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 806                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 807                 if e.cause.code not in (403, 429):
 808                     retry.error = e
 809                     continue
 810                 return self._error_or_warning(e, fatal=fatal)
 811
 812             try:
 813                 self._extract_and_report_alerts(response, only_once=True)
 814             except ExtractorError as e:
 815                 # YouTube servers may return errors we want to retry on in a 200 OK response
 816                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 817                 if 'unknown error' in e.msg.lower():
 818                     retry.error = e
 819                     continue
 820                 return self._error_or_warning(e, fatal=fatal)
 821             # Youtube sometimes sends incomplete data
 822             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 823             if not traverse_obj(response, *variadic(check_get_keys)):
 824                 retry.error = ExtractorError('Incomplete data received', expected=True)
 825                 continue
 826
 827             return response
 828
 829     @staticmethod
 830     def is_music_url(url):
 831         return re.match(r'https?://music\.youtube\.com/', url) is not None
 832
 833     def _extract_video(self, renderer):
 834         video_id = renderer.get('videoId')
 835         title = self._get_text(renderer, 'title')
 836         description = self._get_text(renderer, 'descriptionSnippet')
 837         duration = parse_duration(self._get_text(
 838             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 839         if duration is None:
 840             duration = parse_duration(self._search_regex(
 841                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 842                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 843                 video_id, default=None, group='duration'))
 844
 845         view_count = self._get_count(renderer, 'viewCountText')
 846
 847         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 848         channel_id = traverse_obj(
 849             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 850             expected_type=str, get_all=False)
 851         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 852         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 853         overlay_style = traverse_obj(
 854             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 855             get_all=False, expected_type=str)
 856         badges = self._extract_badges(renderer)
 857         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 858         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 859             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 860             expected_type=str)) or ''
 861         url = f'https://www.youtube.com/watch?v={video_id}'
 862         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 863             url = f'https://www.youtube.com/shorts/{video_id}'
 864
 865         return {
 866             '_type': 'url',
 867             'ie_key': YoutubeIE.ie_key(),
 868             'id': video_id,
 869             'url': url,
 870             'title': title,
 871             'description': description,
 872             'duration': duration,
 873             'view_count': view_count,
 874             'uploader': uploader,
 875             'channel_id': channel_id,
 876             'thumbnails': thumbnails,
 877             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 878                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 879                             else None),
 880             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 881                             else 'was_live' if 'streamed' in time_text.lower()
 882                             else 'is_live' if overlay_style == 'LIVE' or 'live now' in badges
 883                             else None),
 884             'release_timestamp': scheduled_timestamp,
 885             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 886         }
 887
 888
 889 class YoutubeIE(YoutubeBaseInfoExtractor):
 890     IE_DESC = 'YouTube'
 891     _VALID_URL = r"""(?x)^
 892                      (
 893                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 894                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 895                             (?:www\.)?deturl\.com/www\.youtube\.com|
 896                             (?:www\.)?pwnyoutube\.com|
 897                             (?:www\.)?hooktube\.com|
 898                             (?:www\.)?yourepeat\.com|
 899                             tube\.majestyc\.net|
 900                             %(invidious)s|
 901                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 902                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 903                          (?:                                                  # the various things that can precede the ID:
 904                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 905                              |(?:                                             # or the v= param in all its forms
 906                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 907                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 908                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 909                                  v=
 910                              )
 911                          ))
 912                          |(?:
 913                             youtu\.be|                                        # just youtu.be/xxxx
 914                             vid\.plus|                                        # or vid.plus/xxxx
 915                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 916                             %(invidious)s
 917                          )/
 918                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 919                          )
 920                      )?                                                       # all until now is optional -> you can pass the naked ID
 921                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 922                      (?(1).+)?                                                # if we found the ID, everything can follow
 923                      (?:\#|$)""" % {
 924         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 925     }
 926     _EMBED_REGEX = [r'''(?x)
 927         (?:
 928             <iframe[^>]+?src=|
 929             data-video-url=|
 930             <embed[^>]+?src=|
 931             embedSWF\(?:\s*|
 932             <object[^>]+data=|
 933             new\s+SWFObject\(
 934         )
 935         (["\'])
 936             (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
 937             (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
 938         \1''']
 939     _PLAYER_INFO_RE = (
 940         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 941         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 942         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 943     )
 944     _formats = {
 945         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 946         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 947         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 948         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 949         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 950         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 951         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 953         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 954         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 955         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 956         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 957         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 958         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 959         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 960         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 961         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 962         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 963
 964
 965         # 3D videos
 966         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 967         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 968         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 969         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 970         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 971         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 972         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 973
 974         # Apple HTTP Live Streaming
 975         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 976         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 977         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 978         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 979         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 980         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 981         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 982         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 983
 984         # DASH mp4 video
 985         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 986         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 988         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 989         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 990         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 991         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 992         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 993         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 994         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 995         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 996         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 997
 998         # Dash mp4 audio
 999         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1000         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1001         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1002         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1003         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1004         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1005         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1006
1007         # Dash webm
1008         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1009         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1010         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1011         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1012         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1013         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1014         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1015         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1016         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1017         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1018         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1020         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1021         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1022         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1023         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1024         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1025         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1026         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1027         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1028         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1029         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1030
1031         # Dash webm audio
1032         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1033         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1034
1035         # Dash webm audio with opus inside
1036         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1037         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1038         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1039
1040         # RTMP (unnamed)
1041         '_rtmp': {'protocol': 'rtmp'},
1042
1043         # av01 video only formats sometimes served with "unknown" codecs
1044         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1045         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1046         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1047         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1048         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1049         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1050         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1051         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1052     }
1053     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1054
1055     _GEO_BYPASS = False
1056
1057     IE_NAME = 'youtube'
1058     _TESTS = [
1059         {
1060             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1061             'info_dict': {
1062                 'id': 'BaW_jenozKc',
1063                 'ext': 'mp4',
1064                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1065                 'uploader': 'Philipp Hagemeister',
1066                 'uploader_id': 'phihag',
1067                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1068                 'channel': 'Philipp Hagemeister',
1069                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1070                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1071                 'upload_date': '20121002',
1072                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1073                 'categories': ['Science & Technology'],
1074                 'tags': ['youtube-dl'],
1075                 'duration': 10,
1076                 'view_count': int,
1077                 'like_count': int,
1078                 'availability': 'public',
1079                 'playable_in_embed': True,
1080                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1081                 'live_status': 'not_live',
1082                 'age_limit': 0,
1083                 'start_time': 1,
1084                 'end_time': 9,
1085                 'comment_count': int,
1086                 'channel_follower_count': int
1087             }
1088         },
1089         {
1090             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1091             'note': 'Embed-only video (#1746)',
1092             'info_dict': {
1093                 'id': 'yZIXLfi8CZQ',
1094                 'ext': 'mp4',
1095                 'upload_date': '20120608',
1096                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1097                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1098                 'uploader': 'SET India',
1099                 'uploader_id': 'setindia',
1100                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1101                 'age_limit': 18,
1102             },
1103             'skip': 'Private video',
1104         },
1105         {
1106             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1107             'note': 'Use the first video ID in the URL',
1108             'info_dict': {
1109                 'id': 'BaW_jenozKc',
1110                 'ext': 'mp4',
1111                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1112                 'uploader': 'Philipp Hagemeister',
1113                 'uploader_id': 'phihag',
1114                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1115                 'channel': 'Philipp Hagemeister',
1116                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1117                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1118                 'upload_date': '20121002',
1119                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1120                 'categories': ['Science & Technology'],
1121                 'tags': ['youtube-dl'],
1122                 'duration': 10,
1123                 'view_count': int,
1124                 'like_count': int,
1125                 'availability': 'public',
1126                 'playable_in_embed': True,
1127                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1128                 'live_status': 'not_live',
1129                 'age_limit': 0,
1130                 'comment_count': int,
1131                 'channel_follower_count': int
1132             },
1133             'params': {
1134                 'skip_download': True,
1135             },
1136         },
1137         {
1138             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1139             'note': '256k DASH audio (format 141) via DASH manifest',
1140             'info_dict': {
1141                 'id': 'a9LDPn-MO4I',
1142                 'ext': 'm4a',
1143                 'upload_date': '20121002',
1144                 'uploader_id': '8KVIDEO',
1145                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1146                 'description': '',
1147                 'uploader': '8KVIDEO',
1148                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1149             },
1150             'params': {
1151                 'youtube_include_dash_manifest': True,
1152                 'format': '141',
1153             },
1154             'skip': 'format 141 not served anymore',
1155         },
1156         # DASH manifest with encrypted signature
1157         {
1158             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1159             'info_dict': {
1160                 'id': 'IB3lcPjvWLA',
1161                 'ext': 'm4a',
1162                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1163                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1164                 'duration': 244,
1165                 'uploader': 'AfrojackVEVO',
1166                 'uploader_id': 'AfrojackVEVO',
1167                 'upload_date': '20131011',
1168                 'abr': 129.495,
1169                 'like_count': int,
1170                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1171                 'playable_in_embed': True,
1172                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1173                 'view_count': int,
1174                 'track': 'The Spark',
1175                 'live_status': 'not_live',
1176                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1177                 'channel': 'Afrojack',
1178                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1179                 'tags': 'count:19',
1180                 'availability': 'public',
1181                 'categories': ['Music'],
1182                 'age_limit': 0,
1183                 'alt_title': 'The Spark',
1184                 'channel_follower_count': int
1185             },
1186             'params': {
1187                 'youtube_include_dash_manifest': True,
1188                 'format': '141/bestaudio[ext=m4a]',
1189             },
1190         },
1191         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1192         {
1193             'note': 'Embed allowed age-gate video',
1194             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1195             'info_dict': {
1196                 'id': 'HtVdAasjOgU',
1197                 'ext': 'mp4',
1198                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1199                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1200                 'duration': 142,
1201                 'uploader': 'The Witcher',
1202                 'uploader_id': 'WitcherGame',
1203                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1204                 'upload_date': '20140605',
1205                 'age_limit': 18,
1206                 'categories': ['Gaming'],
1207                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1208                 'availability': 'needs_auth',
1209                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1210                 'like_count': int,
1211                 'channel': 'The Witcher',
1212                 'live_status': 'not_live',
1213                 'tags': 'count:17',
1214                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1215                 'playable_in_embed': True,
1216                 'view_count': int,
1217                 'channel_follower_count': int
1218             },
1219         },
1220         {
1221             'note': 'Age-gate video with embed allowed in public site',
1222             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1223             'info_dict': {
1224                 'id': 'HsUATh_Nc2U',
1225                 'ext': 'mp4',
1226                 'title': 'Godzilla 2 (Official Video)',
1227                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1228                 'upload_date': '20200408',
1229                 'uploader_id': 'FlyingKitty900',
1230                 'uploader': 'FlyingKitty',
1231                 'age_limit': 18,
1232                 'availability': 'needs_auth',
1233                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1234                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1235                 'channel': 'FlyingKitty',
1236                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1237                 'view_count': int,
1238                 'categories': ['Entertainment'],
1239                 'live_status': 'not_live',
1240                 'tags': ['Flyingkitty', 'godzilla 2'],
1241                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1242                 'like_count': int,
1243                 'duration': 177,
1244                 'playable_in_embed': True,
1245                 'channel_follower_count': int
1246             },
1247         },
1248         {
1249             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1250             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1251             'info_dict': {
1252                 'id': 'Tq92D6wQ1mg',
1253                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1254                 'ext': 'mp4',
1255                 'upload_date': '20191228',
1256                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1257                 'uploader': 'Projekt Melody',
1258                 'description': 'md5:17eccca93a786d51bc67646756894066',
1259                 'age_limit': 18,
1260                 'like_count': int,
1261                 'availability': 'needs_auth',
1262                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1263                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1264                 'view_count': int,
1265                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1266                 'channel': 'Projekt Melody',
1267                 'live_status': 'not_live',
1268                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1269                 'playable_in_embed': True,
1270                 'categories': ['Entertainment'],
1271                 'duration': 106,
1272                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1273                 'comment_count': int,
1274                 'channel_follower_count': int
1275             },
1276         },
1277         {
1278             'note': 'Non-Agegated non-embeddable video',
1279             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1280             'info_dict': {
1281                 'id': 'MeJVWBSsPAY',
1282                 'ext': 'mp4',
1283                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1284                 'uploader': 'Herr Lurik',
1285                 'uploader_id': 'st3in234',
1286                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1287                 'upload_date': '20130730',
1288                 'track': 'Such mich find mich',
1289                 'age_limit': 0,
1290                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1291                 'like_count': int,
1292                 'playable_in_embed': False,
1293                 'creator': 'OOMPH!',
1294                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1295                 'view_count': int,
1296                 'alt_title': 'Such mich find mich',
1297                 'duration': 210,
1298                 'channel': 'Herr Lurik',
1299                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1300                 'categories': ['Music'],
1301                 'availability': 'public',
1302                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1303                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1304                 'live_status': 'not_live',
1305                 'artist': 'OOMPH!',
1306                 'channel_follower_count': int
1307             },
1308         },
1309         {
1310             'note': 'Non-bypassable age-gated video',
1311             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1312             'only_matching': True,
1313         },
1314         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1315         # YouTube Red ad is not captured for creator
1316         {
1317             'url': '__2ABJjxzNo',
1318             'info_dict': {
1319                 'id': '__2ABJjxzNo',
1320                 'ext': 'mp4',
1321                 'duration': 266,
1322                 'upload_date': '20100430',
1323                 'uploader_id': 'deadmau5',
1324                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1325                 'creator': 'deadmau5',
1326                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1327                 'uploader': 'deadmau5',
1328                 'title': 'Deadmau5 - Some Chords (HD)',
1329                 'alt_title': 'Some Chords',
1330                 'availability': 'public',
1331                 'tags': 'count:14',
1332                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1333                 'view_count': int,
1334                 'live_status': 'not_live',
1335                 'channel': 'deadmau5',
1336                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1337                 'like_count': int,
1338                 'track': 'Some Chords',
1339                 'artist': 'deadmau5',
1340                 'playable_in_embed': True,
1341                 'age_limit': 0,
1342                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1343                 'categories': ['Music'],
1344                 'album': 'Some Chords',
1345                 'channel_follower_count': int
1346             },
1347             'expected_warnings': [
1348                 'DASH manifest missing',
1349             ]
1350         },
1351         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1352         {
1353             'url': 'lqQg6PlCWgI',
1354             'info_dict': {
1355                 'id': 'lqQg6PlCWgI',
1356                 'ext': 'mp4',
1357                 'duration': 6085,
1358                 'upload_date': '20150827',
1359                 'uploader_id': 'olympic',
1360                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1361                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1362                 'uploader': 'Olympics',
1363                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1364                 'like_count': int,
1365                 'release_timestamp': 1343767800,
1366                 'playable_in_embed': True,
1367                 'categories': ['Sports'],
1368                 'release_date': '20120731',
1369                 'channel': 'Olympics',
1370                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1371                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1372                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1373                 'age_limit': 0,
1374                 'availability': 'public',
1375                 'live_status': 'was_live',
1376                 'view_count': int,
1377                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1378                 'channel_follower_count': int
1379             },
1380             'params': {
1381                 'skip_download': 'requires avconv',
1382             }
1383         },
1384         # Non-square pixels
1385         {
1386             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1387             'info_dict': {
1388                 'id': '_b-2C3KPAM0',
1389                 'ext': 'mp4',
1390                 'stretched_ratio': 16 / 9.,
1391                 'duration': 85,
1392                 'upload_date': '20110310',
1393                 'uploader_id': 'AllenMeow',
1394                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1395                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1396                 'uploader': '孫ᄋᄅ',
1397                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1398                 'playable_in_embed': True,
1399                 'channel': '孫ᄋᄅ',
1400                 'age_limit': 0,
1401                 'tags': 'count:11',
1402                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1403                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1404                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1405                 'view_count': int,
1406                 'categories': ['People & Blogs'],
1407                 'like_count': int,
1408                 'live_status': 'not_live',
1409                 'availability': 'unlisted',
1410                 'comment_count': int,
1411                 'channel_follower_count': int
1412             },
1413         },
1414         # url_encoded_fmt_stream_map is empty string
1415         {
1416             'url': 'qEJwOuvDf7I',
1417             'info_dict': {
1418                 'id': 'qEJwOuvDf7I',
1419                 'ext': 'webm',
1420                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1421                 'description': '',
1422                 'upload_date': '20150404',
1423                 'uploader_id': 'spbelect',
1424                 'uploader': 'Наблюдатели Петербурга',
1425             },
1426             'params': {
1427                 'skip_download': 'requires avconv',
1428             },
1429             'skip': 'This live event has ended.',
1430         },
1431         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1432         {
1433             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1434             'info_dict': {
1435                 'id': 'FIl7x6_3R5Y',
1436                 'ext': 'webm',
1437                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1438                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1439                 'duration': 220,
1440                 'upload_date': '20150625',
1441                 'uploader_id': 'dorappi2000',
1442                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1443                 'uploader': 'dorappi2000',
1444                 'formats': 'mincount:31',
1445             },
1446             'skip': 'not actual anymore',
1447         },
1448         # DASH manifest with segment_list
1449         {
1450             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1451             'md5': '8ce563a1d667b599d21064e982ab9e31',
1452             'info_dict': {
1453                 'id': 'CsmdDsKjzN8',
1454                 'ext': 'mp4',
1455                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1456                 'uploader': 'Airtek',
1457                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1458                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1459                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1460             },
1461             'params': {
1462                 'youtube_include_dash_manifest': True,
1463                 'format': '135',  # bestvideo
1464             },
1465             'skip': 'This live event has ended.',
1466         },
1467         {
1468             # Multifeed videos (multiple cameras), URL is for Main Camera
1469             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1470             'info_dict': {
1471                 'id': 'jvGDaLqkpTg',
1472                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1473                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1474             },
1475             'playlist': [{
1476                 'info_dict': {
1477                     'id': 'jvGDaLqkpTg',
1478                     'ext': 'mp4',
1479                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1480                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1481                     'duration': 10643,
1482                     'upload_date': '20161111',
1483                     'uploader': 'Team PGP',
1484                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1485                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1486                 },
1487             }, {
1488                 'info_dict': {
1489                     'id': '3AKt1R1aDnw',
1490                     'ext': 'mp4',
1491                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1492                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1493                     'duration': 10991,
1494                     'upload_date': '20161111',
1495                     'uploader': 'Team PGP',
1496                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1497                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1498                 },
1499             }, {
1500                 'info_dict': {
1501                     'id': 'RtAMM00gpVc',
1502                     'ext': 'mp4',
1503                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1504                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1505                     'duration': 10995,
1506                     'upload_date': '20161111',
1507                     'uploader': 'Team PGP',
1508                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1509                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1510                 },
1511             }, {
1512                 'info_dict': {
1513                     'id': '6N2fdlP3C5U',
1514                     'ext': 'mp4',
1515                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1516                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1517                     'duration': 10990,
1518                     'upload_date': '20161111',
1519                     'uploader': 'Team PGP',
1520                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1521                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1522                 },
1523             }],
1524             'params': {
1525                 'skip_download': True,
1526             },
1527             'skip': 'Not multifeed anymore',
1528         },
1529         {
1530             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1531             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1532             'info_dict': {
1533                 'id': 'gVfLd0zydlo',
1534                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1535             },
1536             'playlist_count': 2,
1537             'skip': 'Not multifeed anymore',
1538         },
1539         {
1540             'url': 'https://vid.plus/FlRa-iH7PGw',
1541             'only_matching': True,
1542         },
1543         {
1544             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1545             'only_matching': True,
1546         },
1547         {
1548             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1549             # Also tests cut-off URL expansion in video description (see
1550             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1551             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1552             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1553             'info_dict': {
1554                 'id': 'lsguqyKfVQg',
1555                 'ext': 'mp4',
1556                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1557                 'alt_title': 'Dark Walk',
1558                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1559                 'duration': 133,
1560                 'upload_date': '20151119',
1561                 'uploader_id': 'IronSoulElf',
1562                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1563                 'uploader': 'IronSoulElf',
1564                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1565                 'track': 'Dark Walk',
1566                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1567                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1568                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1569                 'categories': ['Film & Animation'],
1570                 'view_count': int,
1571                 'live_status': 'not_live',
1572                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1573                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1574                 'tags': 'count:13',
1575                 'availability': 'public',
1576                 'channel': 'IronSoulElf',
1577                 'playable_in_embed': True,
1578                 'like_count': int,
1579                 'age_limit': 0,
1580                 'channel_follower_count': int
1581             },
1582             'params': {
1583                 'skip_download': True,
1584             },
1585         },
1586         {
1587             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1588             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1589             'only_matching': True,
1590         },
1591         {
1592             # Video with yt:stretch=17:0
1593             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1594             'info_dict': {
1595                 'id': 'Q39EVAstoRM',
1596                 'ext': 'mp4',
1597                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1598                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1599                 'upload_date': '20151107',
1600                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1601                 'uploader': 'CH GAMER DROID',
1602             },
1603             'params': {
1604                 'skip_download': True,
1605             },
1606             'skip': 'This video does not exist.',
1607         },
1608         {
1609             # Video with incomplete 'yt:stretch=16:'
1610             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1611             'only_matching': True,
1612         },
1613         {
1614             # Video licensed under Creative Commons
1615             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1616             'info_dict': {
1617                 'id': 'M4gD1WSo5mA',
1618                 'ext': 'mp4',
1619                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1620                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1621                 'duration': 721,
1622                 'upload_date': '20150128',
1623                 'uploader_id': 'BerkmanCenter',
1624                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1625                 'uploader': 'The Berkman Klein Center for Internet & Society',
1626                 'license': 'Creative Commons Attribution license (reuse allowed)',
1627                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1628                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1629                 'like_count': int,
1630                 'age_limit': 0,
1631                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1632                 'channel': 'The Berkman Klein Center for Internet & Society',
1633                 'availability': 'public',
1634                 'view_count': int,
1635                 'categories': ['Education'],
1636                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1637                 'live_status': 'not_live',
1638                 'playable_in_embed': True,
1639                 'comment_count': int,
1640                 'channel_follower_count': int
1641             },
1642             'params': {
1643                 'skip_download': True,
1644             },
1645         },
1646         {
1647             # Channel-like uploader_url
1648             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1649             'info_dict': {
1650                 'id': 'eQcmzGIKrzg',
1651                 'ext': 'mp4',
1652                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1653                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1654                 'duration': 4060,
1655                 'upload_date': '20151120',
1656                 'uploader': 'Bernie Sanders',
1657                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1658                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1659                 'license': 'Creative Commons Attribution license (reuse allowed)',
1660                 'playable_in_embed': True,
1661                 'tags': 'count:12',
1662                 'like_count': int,
1663                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1664                 'age_limit': 0,
1665                 'availability': 'public',
1666                 'categories': ['News & Politics'],
1667                 'channel': 'Bernie Sanders',
1668                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1669                 'view_count': int,
1670                 'live_status': 'not_live',
1671                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1672                 'comment_count': int,
1673                 'channel_follower_count': int
1674             },
1675             'params': {
1676                 'skip_download': True,
1677             },
1678         },
1679         {
1680             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1681             'only_matching': True,
1682         },
1683         {
1684             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1685             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1686             'only_matching': True,
1687         },
1688         {
1689             # Rental video preview
1690             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1691             'info_dict': {
1692                 'id': 'uGpuVWrhIzE',
1693                 'ext': 'mp4',
1694                 'title': 'Piku - Trailer',
1695                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1696                 'upload_date': '20150811',
1697                 'uploader': 'FlixMatrix',
1698                 'uploader_id': 'FlixMatrixKaravan',
1699                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1700                 'license': 'Standard YouTube License',
1701             },
1702             'params': {
1703                 'skip_download': True,
1704             },
1705             'skip': 'This video is not available.',
1706         },
1707         {
1708             # YouTube Red video with episode data
1709             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1710             'info_dict': {
1711                 'id': 'iqKdEhx-dD4',
1712                 'ext': 'mp4',
1713                 'title': 'Isolation - Mind Field (Ep 1)',
1714                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1715                 'duration': 2085,
1716                 'upload_date': '20170118',
1717                 'uploader': 'Vsauce',
1718                 'uploader_id': 'Vsauce',
1719                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1720                 'series': 'Mind Field',
1721                 'season_number': 1,
1722                 'episode_number': 1,
1723                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1724                 'tags': 'count:12',
1725                 'view_count': int,
1726                 'availability': 'public',
1727                 'age_limit': 0,
1728                 'channel': 'Vsauce',
1729                 'episode': 'Episode 1',
1730                 'categories': ['Entertainment'],
1731                 'season': 'Season 1',
1732                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1733                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1734                 'like_count': int,
1735                 'playable_in_embed': True,
1736                 'live_status': 'not_live',
1737                 'channel_follower_count': int
1738             },
1739             'params': {
1740                 'skip_download': True,
1741             },
1742             'expected_warnings': [
1743                 'Skipping DASH manifest',
1744             ],
1745         },
1746         {
1747             # The following content has been identified by the YouTube community
1748             # as inappropriate or offensive to some audiences.
1749             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1750             'info_dict': {
1751                 'id': '6SJNVb0GnPI',
1752                 'ext': 'mp4',
1753                 'title': 'Race Differences in Intelligence',
1754                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1755                 'duration': 965,
1756                 'upload_date': '20140124',
1757                 'uploader': 'New Century Foundation',
1758                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1759                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1760             },
1761             'params': {
1762                 'skip_download': True,
1763             },
1764             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1765         },
1766         {
1767             # itag 212
1768             'url': '1t24XAntNCY',
1769             'only_matching': True,
1770         },
1771         {
1772             # geo restricted to JP
1773             'url': 'sJL6WA-aGkQ',
1774             'only_matching': True,
1775         },
1776         {
1777             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1778             'only_matching': True,
1779         },
1780         {
1781             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1782             'only_matching': True,
1783         },
1784         {
1785             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1786             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1787             'only_matching': True,
1788         },
1789         {
1790             # DRM protected
1791             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1792             'only_matching': True,
1793         },
1794         {
1795             # Video with unsupported adaptive stream type formats
1796             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1797             'info_dict': {
1798                 'id': 'Z4Vy8R84T1U',
1799                 'ext': 'mp4',
1800                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1801                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1802                 'duration': 433,
1803                 'upload_date': '20130923',
1804                 'uploader': 'Amelia Putri Harwita',
1805                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1806                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1807                 'formats': 'maxcount:10',
1808             },
1809             'params': {
1810                 'skip_download': True,
1811                 'youtube_include_dash_manifest': False,
1812             },
1813             'skip': 'not actual anymore',
1814         },
1815         {
1816             # Youtube Music Auto-generated description
1817             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1818             'info_dict': {
1819                 'id': 'MgNrAu2pzNs',
1820                 'ext': 'mp4',
1821                 'title': 'Voyeur Girl',
1822                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1823                 'upload_date': '20190312',
1824                 'uploader': 'Stephen - Topic',
1825                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1826                 'artist': 'Stephen',
1827                 'track': 'Voyeur Girl',
1828                 'album': 'it\'s too much love to know my dear',
1829                 'release_date': '20190313',
1830                 'release_year': 2019,
1831                 'alt_title': 'Voyeur Girl',
1832                 'view_count': int,
1833                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1834                 'playable_in_embed': True,
1835                 'like_count': int,
1836                 'categories': ['Music'],
1837                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1838                 'channel': 'Stephen',
1839                 'availability': 'public',
1840                 'creator': 'Stephen',
1841                 'duration': 169,
1842                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1843                 'age_limit': 0,
1844                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1845                 'tags': 'count:11',
1846                 'live_status': 'not_live',
1847                 'channel_follower_count': int
1848             },
1849             'params': {
1850                 'skip_download': True,
1851             },
1852         },
1853         {
1854             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1855             'only_matching': True,
1856         },
1857         {
1858             # invalid -> valid video id redirection
1859             'url': 'DJztXj2GPfl',
1860             'info_dict': {
1861                 'id': 'DJztXj2GPfk',
1862                 'ext': 'mp4',
1863                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1864                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1865                 'upload_date': '20090125',
1866                 'uploader': 'Prochorowka',
1867                 'uploader_id': 'Prochorowka',
1868                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1869                 'artist': 'Panjabi MC',
1870                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1871                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1872             },
1873             'params': {
1874                 'skip_download': True,
1875             },
1876             'skip': 'Video unavailable',
1877         },
1878         {
1879             # empty description results in an empty string
1880             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1881             'info_dict': {
1882                 'id': 'x41yOUIvK2k',
1883                 'ext': 'mp4',
1884                 'title': 'IMG 3456',
1885                 'description': '',
1886                 'upload_date': '20170613',
1887                 'uploader_id': 'ElevageOrVert',
1888                 'uploader': 'ElevageOrVert',
1889                 'view_count': int,
1890                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1891                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1892                 'like_count': int,
1893                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1894                 'tags': [],
1895                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1896                 'availability': 'public',
1897                 'age_limit': 0,
1898                 'categories': ['Pets & Animals'],
1899                 'duration': 7,
1900                 'playable_in_embed': True,
1901                 'live_status': 'not_live',
1902                 'channel': 'ElevageOrVert',
1903                 'channel_follower_count': int
1904             },
1905             'params': {
1906                 'skip_download': True,
1907             },
1908         },
1909         {
1910             # with '};' inside yt initial data (see [1])
1911             # see [2] for an example with '};' inside ytInitialPlayerResponse
1912             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1913             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1914             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1915             'info_dict': {
1916                 'id': 'CHqg6qOn4no',
1917                 'ext': 'mp4',
1918                 'title': 'Part 77   Sort a list of simple types in c#',
1919                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1920                 'upload_date': '20130831',
1921                 'uploader_id': 'kudvenkat',
1922                 'uploader': 'kudvenkat',
1923                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1924                 'like_count': int,
1925                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1926                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1927                 'live_status': 'not_live',
1928                 'categories': ['Education'],
1929                 'availability': 'public',
1930                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1931                 'tags': 'count:12',
1932                 'playable_in_embed': True,
1933                 'age_limit': 0,
1934                 'view_count': int,
1935                 'duration': 522,
1936                 'channel': 'kudvenkat',
1937                 'comment_count': int,
1938                 'channel_follower_count': int
1939             },
1940             'params': {
1941                 'skip_download': True,
1942             },
1943         },
1944         {
1945             # another example of '};' in ytInitialData
1946             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1947             'only_matching': True,
1948         },
1949         {
1950             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1951             'only_matching': True,
1952         },
1953         {
1954             # https://github.com/ytdl-org/youtube-dl/pull/28094
1955             'url': 'OtqTfy26tG0',
1956             'info_dict': {
1957                 'id': 'OtqTfy26tG0',
1958                 'ext': 'mp4',
1959                 'title': 'Burn Out',
1960                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1961                 'upload_date': '20141120',
1962                 'uploader': 'The Cinematic Orchestra - Topic',
1963                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1964                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1965                 'artist': 'The Cinematic Orchestra',
1966                 'track': 'Burn Out',
1967                 'album': 'Every Day',
1968                 'like_count': int,
1969                 'live_status': 'not_live',
1970                 'alt_title': 'Burn Out',
1971                 'duration': 614,
1972                 'age_limit': 0,
1973                 'view_count': int,
1974                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1975                 'creator': 'The Cinematic Orchestra',
1976                 'channel': 'The Cinematic Orchestra',
1977                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1978                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1979                 'availability': 'public',
1980                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1981                 'categories': ['Music'],
1982                 'playable_in_embed': True,
1983                 'channel_follower_count': int
1984             },
1985             'params': {
1986                 'skip_download': True,
1987             },
1988         },
1989         {
1990             # controversial video, only works with bpctr when authenticated with cookies
1991             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1992             'only_matching': True,
1993         },
1994         {
1995             # controversial video, requires bpctr/contentCheckOk
1996             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1997             'info_dict': {
1998                 'id': 'SZJvDhaSDnc',
1999                 'ext': 'mp4',
2000                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2001                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2002                 'uploader': 'CBS Mornings',
2003                 'uploader_id': 'CBSThisMorning',
2004                 'upload_date': '20140716',
2005                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2006                 'duration': 170,
2007                 'categories': ['News & Politics'],
2008                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2009                 'view_count': int,
2010                 'channel': 'CBS Mornings',
2011                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2012                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2013                 'age_limit': 18,
2014                 'availability': 'needs_auth',
2015                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2016                 'like_count': int,
2017                 'live_status': 'not_live',
2018                 'playable_in_embed': True,
2019                 'channel_follower_count': int
2020             }
2021         },
2022         {
2023             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2024             'url': 'cBvYw8_A0vQ',
2025             'info_dict': {
2026                 'id': 'cBvYw8_A0vQ',
2027                 'ext': 'mp4',
2028                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2029                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2030                 'upload_date': '20201120',
2031                 'uploader': 'Walk around Japan',
2032                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2033                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2034                 'duration': 1456,
2035                 'categories': ['Travel & Events'],
2036                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2037                 'view_count': int,
2038                 'channel': 'Walk around Japan',
2039                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2040                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2041                 'age_limit': 0,
2042                 'availability': 'public',
2043                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2044                 'live_status': 'not_live',
2045                 'playable_in_embed': True,
2046                 'channel_follower_count': int
2047             },
2048             'params': {
2049                 'skip_download': True,
2050             },
2051         }, {
2052             # Has multiple audio streams
2053             'url': 'WaOKSUlf4TM',
2054             'only_matching': True
2055         }, {
2056             # Requires Premium: has format 141 when requested using YTM url
2057             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2058             'only_matching': True
2059         }, {
2060             # multiple subtitles with same lang_code
2061             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2062             'only_matching': True,
2063         }, {
2064             # Force use android client fallback
2065             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2066             'info_dict': {
2067                 'id': 'YOelRv7fMxY',
2068                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2069                 'ext': '3gp',
2070                 'upload_date': '20210624',
2071                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2072                 'uploader': 'colinfurze',
2073                 'uploader_id': 'colinfurze',
2074                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2075                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2076                 'duration': 596,
2077                 'categories': ['Entertainment'],
2078                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2079                 'view_count': int,
2080                 'channel': 'colinfurze',
2081                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2082                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2083                 'age_limit': 0,
2084                 'availability': 'public',
2085                 'like_count': int,
2086                 'live_status': 'not_live',
2087                 'playable_in_embed': True,
2088                 'channel_follower_count': int
2089             },
2090             'params': {
2091                 'format': '17',  # 3gp format available on android
2092                 'extractor_args': {'youtube': {'player_client': ['android']}},
2093             },
2094         },
2095         {
2096             # Skip download of additional client configs (remix client config in this case)
2097             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2098             'only_matching': True,
2099             'params': {
2100                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2101             },
2102         }, {
2103             # shorts
2104             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2105             'only_matching': True,
2106         }, {
2107             'note': 'Storyboards',
2108             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2109             'info_dict': {
2110                 'id': '5KLPxDtMqe8',
2111                 'ext': 'mhtml',
2112                 'format_id': 'sb0',
2113                 'title': 'Your Brain is Plastic',
2114                 'uploader_id': 'scishow',
2115                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2116                 'upload_date': '20140324',
2117                 'uploader': 'SciShow',
2118                 'like_count': int,
2119                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2120                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2121                 'view_count': int,
2122                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2123                 'playable_in_embed': True,
2124                 'tags': 'count:12',
2125                 'uploader_url': 'http://www.youtube.com/user/scishow',
2126                 'availability': 'public',
2127                 'channel': 'SciShow',
2128                 'live_status': 'not_live',
2129                 'duration': 248,
2130                 'categories': ['Education'],
2131                 'age_limit': 0,
2132                 'channel_follower_count': int
2133             }, 'params': {'format': 'mhtml', 'skip_download': True}
2134         }, {
2135             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2136             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2137             'info_dict': {
2138                 'id': '2NUZ8W2llS4',
2139                 'ext': 'mp4',
2140                 'title': 'The NP that test your phone performance 🙂',
2141                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2142                 'uploader': 'Leon Nguyen',
2143                 'uploader_id': 'VNSXIII',
2144                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2145                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2146                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2147                 'duration': 21,
2148                 'view_count': int,
2149                 'age_limit': 0,
2150                 'categories': ['Gaming'],
2151                 'tags': 'count:23',
2152                 'playable_in_embed': True,
2153                 'live_status': 'not_live',
2154                 'upload_date': '20220103',
2155                 'like_count': int,
2156                 'availability': 'public',
2157                 'channel': 'Leon Nguyen',
2158                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2159                 'comment_count': int,
2160                 'channel_follower_count': int
2161             }
2162         }, {
2163             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2164             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2165             'info_dict': {
2166                 'id': '2NUZ8W2llS4',
2167                 'ext': 'mp4',
2168                 'title': 'The NP that test your phone performance 🙂',
2169                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2170                 'uploader': 'Leon Nguyen',
2171                 'uploader_id': 'VNSXIII',
2172                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2173                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2174                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2175                 'duration': 21,
2176                 'view_count': int,
2177                 'age_limit': 0,
2178                 'categories': ['Gaming'],
2179                 'tags': 'count:23',
2180                 'playable_in_embed': True,
2181                 'live_status': 'not_live',
2182                 'upload_date': '20220102',
2183                 'like_count': int,
2184                 'availability': 'public',
2185                 'channel': 'Leon Nguyen',
2186                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2187                 'comment_count': int,
2188                 'channel_follower_count': int
2189             },
2190             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2191         }, {
2192             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2193             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2194             'info_dict': {
2195                 'id': 'mzZzzBU6lrM',
2196                 'ext': 'mp4',
2197                 'title': 'I Met GeorgeNotFound In Real Life...',
2198                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2199                 'uploader': 'Quackity',
2200                 'uploader_id': 'QuackityHQ',
2201                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2202                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2203                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2204                 'duration': 955,
2205                 'view_count': int,
2206                 'age_limit': 0,
2207                 'categories': ['Entertainment'],
2208                 'tags': 'count:26',
2209                 'playable_in_embed': True,
2210                 'live_status': 'not_live',
2211                 'release_timestamp': 1641172509,
2212                 'release_date': '20220103',
2213                 'upload_date': '20220103',
2214                 'like_count': int,
2215                 'availability': 'public',
2216                 'channel': 'Quackity',
2217                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2218                 'channel_follower_count': int
2219             }
2220         },
2221         {   # continuous livestream. Microformat upload date should be preferred.
2222             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2223             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2224             'info_dict': {
2225                 'id': 'kgx4WGK0oNU',
2226                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2227                 'ext': 'mp4',
2228                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2229                 'availability': 'public',
2230                 'age_limit': 0,
2231                 'release_timestamp': 1637975704,
2232                 'upload_date': '20210619',
2233                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2234                 'live_status': 'is_live',
2235                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2236                 'uploader': '阿鲍Abao',
2237                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2238                 'channel': 'Abao in Tokyo',
2239                 'channel_follower_count': int,
2240                 'release_date': '20211127',
2241                 'tags': 'count:39',
2242                 'categories': ['People & Blogs'],
2243                 'like_count': int,
2244                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2245                 'view_count': int,
2246                 'playable_in_embed': True,
2247                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2248             },
2249             'params': {'skip_download': True}
2250         }, {
2251             # Story. Requires specific player params to work.
2252             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2253             'info_dict': {
2254                 'id': 'vv8qTUWmulI',
2255                 'ext': 'mp4',
2256                 'availability': 'unlisted',
2257                 'view_count': int,
2258                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2259                 'upload_date': '20220526',
2260                 'categories': ['Education'],
2261                 'title': 'Story',
2262                 'channel': 'IT\'S HISTORY',
2263                 'description': '',
2264                 'uploader_id': 'BlastfromthePast',
2265                 'duration': 12,
2266                 'uploader': 'IT\'S HISTORY',
2267                 'playable_in_embed': True,
2268                 'age_limit': 0,
2269                 'live_status': 'not_live',
2270                 'tags': [],
2271                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2272                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2273                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2274             },
2275             'skip': 'stories get removed after some period of time',
2276         }, {
2277             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2278             'info_dict': {
2279                 'id': 'tjjjtzRLHvA',
2280                 'ext': 'mp4',
2281                 'title': 'ハッシュタグ無し };if window.ytcsi',
2282                 'upload_date': '20220323',
2283                 'like_count': int,
2284                 'availability': 'unlisted',
2285                 'channel': 'nao20010128nao',
2286                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2287                 'age_limit': 0,
2288                 'uploader': 'nao20010128nao',
2289                 'uploader_id': 'nao20010128nao',
2290                 'categories': ['Music'],
2291                 'view_count': int,
2292                 'description': '',
2293                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2294                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2295                 'live_status': 'not_live',
2296                 'playable_in_embed': True,
2297                 'channel_follower_count': int,
2298                 'duration': 6,
2299                 'tags': [],
2300                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2301             }
2302         }, {
2303             'note': '6 channel audio',
2304             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2305             'only_matching': True,
2306         }
2307     ]
2308
2309     _WEBPAGE_TESTS = [
2310         # YouTube <object> embed
2311         {
2312             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2313             'md5': '873c81d308b979f0e23ee7e620b312a3',
2314             'info_dict': {
2315                 'id': 'msN87y-iEx0',
2316                 'ext': 'mp4',
2317                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2318                 'upload_date': '20080526',
2319                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2320                 'uploader': 'Christopher Sykes',
2321                 'uploader_id': 'ChristopherJSykes',
2322                 'age_limit': 0,
2323                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2324                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2325                 'playable_in_embed': True,
2326                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2327                 'like_count': int,
2328                 'comment_count': int,
2329                 'channel': 'Christopher Sykes',
2330                 'live_status': 'not_live',
2331                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2332                 'availability': 'public',
2333                 'duration': 195,
2334                 'view_count': int,
2335                 'categories': ['Science & Technology'],
2336                 'channel_follower_count': int,
2337                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2338             },
2339             'params': {
2340                 'skip_download': True,
2341             }
2342         },
2343     ]
2344
2345     @classmethod
2346     def suitable(cls, url):
2347         from ..utils import parse_qs
2348
2349         qs = parse_qs(url)
2350         if qs.get('list', [None])[0]:
2351             return False
2352         return super().suitable(url)
2353
2354     def __init__(self, *args, **kwargs):
2355         super().__init__(*args, **kwargs)
2356         self._code_cache = {}
2357         self._player_cache = {}
2358
2359     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2360         lock = threading.Lock()
2361
2362         is_live = True
2363         start_time = time.time()
2364         formats = [f for f in formats if f.get('is_from_start')]
2365
2366         def refetch_manifest(format_id, delay):
2367             nonlocal formats, start_time, is_live
2368             if time.time() <= start_time + delay:
2369                 return
2370
2371             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2372             video_details = traverse_obj(
2373                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2374             microformats = traverse_obj(
2375                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2376                 expected_type=dict, default=[])
2377             _, is_live, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2378             start_time = time.time()
2379
2380         def mpd_feed(format_id, delay):
2381             """
2382             @returns (manifest_url, manifest_stream_number, is_live) or None
2383             """
2384             with lock:
2385                 refetch_manifest(format_id, delay)
2386
2387             f = next((f for f in formats if f['format_id'] == format_id), None)
2388             if not f:
2389                 if not is_live:
2390                     self.to_screen(f'{video_id}: Video is no longer live')
2391                 else:
2392                     self.report_warning(
2393                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2394                 return None
2395             return f['manifest_url'], f['manifest_stream_number'], is_live
2396
2397         for f in formats:
2398             f['is_live'] = True
2399             f['protocol'] = 'http_dash_segments_generator'
2400             f['fragments'] = functools.partial(
2401                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2402
2403     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2404         FETCH_SPAN, MAX_DURATION = 5, 432000
2405
2406         mpd_url, stream_number, is_live = None, None, True
2407
2408         begin_index = 0
2409         download_start_time = ctx.get('start') or time.time()
2410
2411         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2412         if lack_early_segments:
2413             self.report_warning(bug_reports_message(
2414                 'Starting download from the last 120 hours of the live stream since '
2415                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2416             lack_early_segments = True
2417
2418         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2419         fragments, fragment_base_url = None, None
2420
2421         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2422             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2423             # Obtain from MPD's maximum seq value
2424             old_mpd_url = mpd_url
2425             last_error = ctx.pop('last_error', None)
2426             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2427             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2428                                                or (mpd_url, stream_number, False))
2429             if not refresh_sequence:
2430                 if expire_fast and not is_live:
2431                     return False, last_seq
2432                 elif old_mpd_url == mpd_url:
2433                     return True, last_seq
2434             try:
2435                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2436                     mpd_url, None, note=False, errnote=False, fatal=False)
2437             except ExtractorError:
2438                 fmts = None
2439             if not fmts:
2440                 no_fragment_score += 2
2441                 return False, last_seq
2442             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2443             fragments = fmt_info['fragments']
2444             fragment_base_url = fmt_info['fragment_base_url']
2445             assert fragment_base_url
2446
2447             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2448             return True, _last_seq
2449
2450         while is_live:
2451             fetch_time = time.time()
2452             if no_fragment_score > 30:
2453                 return
2454             if last_segment_url:
2455                 # Obtain from "X-Head-Seqnum" header value from each segment
2456                 try:
2457                     urlh = self._request_webpage(
2458                         last_segment_url, None, note=False, errnote=False, fatal=False)
2459                 except ExtractorError:
2460                     urlh = None
2461                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2462                 if last_seq is None:
2463                     no_fragment_score += 2
2464                     last_segment_url = None
2465                     continue
2466             else:
2467                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2468                 no_fragment_score += 2
2469                 if not should_continue:
2470                     continue
2471
2472             if known_idx > last_seq:
2473                 last_segment_url = None
2474                 continue
2475
2476             last_seq += 1
2477
2478             if begin_index < 0 and known_idx < 0:
2479                 # skip from the start when it's negative value
2480                 known_idx = last_seq + begin_index
2481             if lack_early_segments:
2482                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2483             try:
2484                 for idx in range(known_idx, last_seq):
2485                     # do not update sequence here or you'll get skipped some part of it
2486                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2487                     if not should_continue:
2488                         known_idx = idx - 1
2489                         raise ExtractorError('breaking out of outer loop')
2490                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2491                     yield {
2492                         'url': last_segment_url,
2493                         'fragment_count': last_seq,
2494                     }
2495                 if known_idx == last_seq:
2496                     no_fragment_score += 5
2497                 else:
2498                     no_fragment_score = 0
2499                 known_idx = last_seq
2500             except ExtractorError:
2501                 continue
2502
2503             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2504
2505     def _extract_player_url(self, *ytcfgs, webpage=None):
2506         player_url = traverse_obj(
2507             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2508             get_all=False, expected_type=str)
2509         if not player_url:
2510             return
2511         return urljoin('https://www.youtube.com', player_url)
2512
2513     def _download_player_url(self, video_id, fatal=False):
2514         res = self._download_webpage(
2515             'https://www.youtube.com/iframe_api',
2516             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2517         if res:
2518             player_version = self._search_regex(
2519                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2520             if player_version:
2521                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2522
2523     def _signature_cache_id(self, example_sig):
2524         """ Return a string representation of a signature """
2525         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2526
2527     @classmethod
2528     def _extract_player_info(cls, player_url):
2529         for player_re in cls._PLAYER_INFO_RE:
2530             id_m = re.search(player_re, player_url)
2531             if id_m:
2532                 break
2533         else:
2534             raise ExtractorError('Cannot identify player %r' % player_url)
2535         return id_m.group('id')
2536
2537     def _load_player(self, video_id, player_url, fatal=True):
2538         player_id = self._extract_player_info(player_url)
2539         if player_id not in self._code_cache:
2540             code = self._download_webpage(
2541                 player_url, video_id, fatal=fatal,
2542                 note='Downloading player ' + player_id,
2543                 errnote='Download of %s failed' % player_url)
2544             if code:
2545                 self._code_cache[player_id] = code
2546         return self._code_cache.get(player_id)
2547
2548     def _extract_signature_function(self, video_id, player_url, example_sig):
2549         player_id = self._extract_player_info(player_url)
2550
2551         # Read from filesystem cache
2552         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2553         assert os.path.basename(func_id) == func_id
2554
2555         self.write_debug(f'Extracting signature function {func_id}')
2556         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2557
2558         if not cache_spec:
2559             code = self._load_player(video_id, player_url)
2560         if code:
2561             res = self._parse_sig_js(code)
2562             test_string = ''.join(map(chr, range(len(example_sig))))
2563             cache_spec = [ord(c) for c in res(test_string)]
2564             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2565
2566         return lambda s: ''.join(s[i] for i in cache_spec)
2567
2568     def _print_sig_code(self, func, example_sig):
2569         if not self.get_param('youtube_print_sig_code'):
2570             return
2571
2572         def gen_sig_code(idxs):
2573             def _genslice(start, end, step):
2574                 starts = '' if start == 0 else str(start)
2575                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2576                 steps = '' if step == 1 else (':%d' % step)
2577                 return f's[{starts}{ends}{steps}]'
2578
2579             step = None
2580             # Quelch pyflakes warnings - start will be set when step is set
2581             start = '(Never used)'
2582             for i, prev in zip(idxs[1:], idxs[:-1]):
2583                 if step is not None:
2584                     if i - prev == step:
2585                         continue
2586                     yield _genslice(start, prev, step)
2587                     step = None
2588                     continue
2589                 if i - prev in [-1, 1]:
2590                     step = i - prev
2591                     start = prev
2592                     continue
2593                 else:
2594                     yield 's[%d]' % prev
2595             if step is None:
2596                 yield 's[%d]' % i
2597             else:
2598                 yield _genslice(start, i, step)
2599
2600         test_string = ''.join(map(chr, range(len(example_sig))))
2601         cache_res = func(test_string)
2602         cache_spec = [ord(c) for c in cache_res]
2603         expr_code = ' + '.join(gen_sig_code(cache_spec))
2604         signature_id_tuple = '(%s)' % (
2605             ', '.join(str(len(p)) for p in example_sig.split('.')))
2606         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2607                 '    return %s\n') % (signature_id_tuple, expr_code)
2608         self.to_screen('Extracted signature function:\n' + code)
2609
2610     def _parse_sig_js(self, jscode):
2611         funcname = self._search_regex(
2612             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2613              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2614              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2615              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2616              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2617              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2618              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2619              # Obsolete patterns
2620              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2621              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2622              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2623              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2624              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2625              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2626              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2627              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2628             jscode, 'Initial JS player signature function name', group='sig')
2629
2630         jsi = JSInterpreter(jscode)
2631         initial_function = jsi.extract_function(funcname)
2632         return lambda s: initial_function([s])
2633
2634     def _cached(self, func, *cache_id):
2635         def inner(*args, **kwargs):
2636             if cache_id not in self._player_cache:
2637                 try:
2638                     self._player_cache[cache_id] = func(*args, **kwargs)
2639                 except ExtractorError as e:
2640                     self._player_cache[cache_id] = e
2641                 except Exception as e:
2642                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
2643
2644             ret = self._player_cache[cache_id]
2645             if isinstance(ret, Exception):
2646                 raise ret
2647             return ret
2648         return inner
2649
2650     def _decrypt_signature(self, s, video_id, player_url):
2651         """Turn the encrypted s field into a working signature"""
2652         extract_sig = self._cached(
2653             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
2654         func = extract_sig(video_id, player_url, s)
2655         self._print_sig_code(func, s)
2656         return func(s)
2657
2658     def _decrypt_nsig(self, s, video_id, player_url):
2659         """Turn the encrypted n field into a working signature"""
2660         if player_url is None:
2661             raise ExtractorError('Cannot decrypt nsig without player_url')
2662         player_url = urljoin('https://www.youtube.com', player_url)
2663
2664         jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
2665         if self.get_param('youtube_print_sig_code'):
2666             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2667
2668         try:
2669             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
2670             ret = extract_nsig(jsi, func_code)(s)
2671         except JSInterpreter.Exception as e:
2672             try:
2673                 jsi = PhantomJSwrapper(self, timeout=5000)
2674             except ExtractorError:
2675                 raise e
2676             self.report_warning(
2677                 f'Native nsig extraction failed: Trying with PhantomJS\n'
2678                 f'         n = {s} ; player = {player_url}', video_id)
2679             self.write_debug(e)
2680
2681             args, func_body = func_code
2682             ret = jsi.execute(
2683                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
2684                 video_id=video_id, note='Executing signature code').strip()
2685
2686         self.write_debug(f'Decrypted nsig {s} => {ret}')
2687         return ret
2688
2689     def _extract_n_function_name(self, jscode):
2690         funcname, idx = self._search_regex(
2691             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2692             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2693         if not idx:
2694             return funcname
2695
2696         return json.loads(js_to_json(self._search_regex(
2697             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
2698             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
2699
2700     def _extract_n_function_code(self, video_id, player_url):
2701         player_id = self._extract_player_info(player_url)
2702         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.08.19.2')
2703         jscode = func_code or self._load_player(video_id, player_url)
2704         jsi = JSInterpreter(jscode)
2705
2706         if func_code:
2707             return jsi, player_id, func_code
2708
2709         func_code = jsi.extract_function_code(self._extract_n_function_name(jscode))
2710         self.cache.store('youtube-nsig', player_id, func_code)
2711         return jsi, player_id, func_code
2712
2713     def _extract_n_function_from_code(self, jsi, func_code):
2714         func = jsi.extract_function_from_code(*func_code)
2715
2716         def extract_nsig(s):
2717             try:
2718                 ret = func([s])
2719             except JSInterpreter.Exception:
2720                 raise
2721             except Exception as e:
2722                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
2723
2724             if ret.startswith('enhanced_except_'):
2725                 raise JSInterpreter.Exception('Signature function returned an exception')
2726             return ret
2727
2728         return extract_nsig
2729
2730     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2731         """
2732         Extract signatureTimestamp (sts)
2733         Required to tell API what sig/player version is in use.
2734         """
2735         sts = None
2736         if isinstance(ytcfg, dict):
2737             sts = int_or_none(ytcfg.get('STS'))
2738
2739         if not sts:
2740             # Attempt to extract from player
2741             if player_url is None:
2742                 error_msg = 'Cannot extract signature timestamp without player_url.'
2743                 if fatal:
2744                     raise ExtractorError(error_msg)
2745                 self.report_warning(error_msg)
2746                 return
2747             code = self._load_player(video_id, player_url, fatal=fatal)
2748             if code:
2749                 sts = int_or_none(self._search_regex(
2750                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2751                     'JS player signature timestamp', group='sts', fatal=fatal))
2752         return sts
2753
2754     def _mark_watched(self, video_id, player_responses):
2755         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
2756             label = 'fully ' if is_full else ''
2757             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
2758                             expected_type=url_or_none)
2759             if not url:
2760                 self.report_warning(f'Unable to mark {label}watched')
2761                 return
2762             parsed_url = urllib.parse.urlparse(url)
2763             qs = urllib.parse.parse_qs(parsed_url.query)
2764
2765             # cpn generation algorithm is reverse engineered from base.js.
2766             # In fact it works even with dummy cpn.
2767             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2768             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2769
2770             # # more consistent results setting it to right before the end
2771             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
2772
2773             qs.update({
2774                 'ver': ['2'],
2775                 'cpn': [cpn],
2776                 'cmt': video_length,
2777                 'el': 'detailpage',  # otherwise defaults to "shorts"
2778             })
2779
2780             if is_full:
2781                 # these seem to mark watchtime "history" in the real world
2782                 # they're required, so send in a single value
2783                 qs.update({
2784                     'st': video_length,
2785                     'et': video_length,
2786                 })
2787
2788             url = urllib.parse.urlunparse(
2789                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
2790
2791             self._download_webpage(
2792                 url, video_id, f'Marking {label}watched',
2793                 'Unable to mark watched', fatal=False)
2794
2795     @classmethod
2796     def _extract_from_webpage(cls, url, webpage):
2797         # Invidious Instances
2798         # https://github.com/yt-dlp/yt-dlp/issues/195
2799         # https://github.com/iv-org/invidious/pull/1730
2800         mobj = re.search(
2801             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
2802             webpage)
2803         if mobj:
2804             yield cls.url_result(mobj.group('url'), cls)
2805             raise cls.StopExtraction()
2806
2807         yield from super()._extract_from_webpage(url, webpage)
2808
2809         # lazyYT YouTube embed
2810         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
2811             yield cls.url_result(unescapeHTML(id_), cls, id_)
2812
2813         # Wordpress "YouTube Video Importer" plugin
2814         for m in re.findall(r'''(?x)<div[^>]+
2815                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2816                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
2817             yield cls.url_result(m[-1], cls, m[-1])
2818
2819     @classmethod
2820     def extract_id(cls, url):
2821         video_id = cls.get_temp_id(url)
2822         if not video_id:
2823             raise ExtractorError(f'Invalid URL: {url}')
2824         return video_id
2825
2826     def _extract_chapters_from_json(self, data, duration):
2827         chapter_list = traverse_obj(
2828             data, (
2829                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2830                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2831             ), expected_type=list)
2832
2833         return self._extract_chapters(
2834             chapter_list,
2835             chapter_time=lambda chapter: float_or_none(
2836                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2837             chapter_title=lambda chapter: traverse_obj(
2838                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2839             duration=duration)
2840
2841     def _extract_chapters_from_engagement_panel(self, data, duration):
2842         content_list = traverse_obj(
2843             data,
2844             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2845             expected_type=list, default=[])
2846         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2847         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2848
2849         return next(filter(None, (
2850             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2851                                    chapter_time, chapter_title, duration)
2852             for contents in content_list)), [])
2853
2854     def _extract_chapters_from_description(self, description, duration):
2855         return self._extract_chapters(
2856             re.findall(r'(?m)^((?:\d+:)?\d{1,2}:\d{2})\b\W*\s(.+?)\s*$', description or ''),
2857             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
2858             duration=duration, strict=False)
2859
2860     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
2861         if not duration:
2862             return
2863         chapter_list = [{
2864             'start_time': chapter_time(chapter),
2865             'title': chapter_title(chapter),
2866         } for chapter in chapter_list or []]
2867         if not strict:
2868             chapter_list.sort(key=lambda c: c['start_time'] or 0)
2869
2870         chapters = [{'start_time': 0}]
2871         for idx, chapter in enumerate(chapter_list):
2872             if chapter['start_time'] is None:
2873                 self.report_warning(f'Incomplete chapter {idx}')
2874             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
2875                 chapters.append(chapter)
2876             else:
2877                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
2878         return chapters[1:]
2879
2880     def _extract_comment(self, comment_renderer, parent=None):
2881         comment_id = comment_renderer.get('commentId')
2882         if not comment_id:
2883             return
2884
2885         text = self._get_text(comment_renderer, 'contentText')
2886
2887         # note: timestamp is an estimate calculated from the current time and time_text
2888         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2889         author = self._get_text(comment_renderer, 'authorText')
2890         author_id = try_get(comment_renderer,
2891                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
2892
2893         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2894                                                        lambda x: x['likeCount']), str)) or 0
2895         author_thumbnail = try_get(comment_renderer,
2896                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
2897
2898         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2899         is_favorited = 'creatorHeart' in (try_get(
2900             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2901         return {
2902             'id': comment_id,
2903             'text': text,
2904             'timestamp': timestamp,
2905             'time_text': time_text,
2906             'like_count': votes,
2907             'is_favorited': is_favorited,
2908             'author': author,
2909             'author_id': author_id,
2910             'author_thumbnail': author_thumbnail,
2911             'author_is_uploader': author_is_uploader,
2912             'parent': parent or 'root'
2913         }
2914
2915     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2916
2917         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2918
2919         def extract_header(contents):
2920             _continuation = None
2921             for content in contents:
2922                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2923                 expected_comment_count = self._get_count(
2924                     comments_header_renderer, 'countText', 'commentsCount')
2925
2926                 if expected_comment_count:
2927                     tracker['est_total'] = expected_comment_count
2928                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2929                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2930
2931                 sort_menu_item = try_get(
2932                     comments_header_renderer,
2933                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2934                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2935
2936                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2937                 if not _continuation:
2938                     continue
2939
2940                 sort_text = str_or_none(sort_menu_item.get('title'))
2941                 if not sort_text:
2942                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2943                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2944                 break
2945             return _continuation
2946
2947         def extract_thread(contents):
2948             if not parent:
2949                 tracker['current_page_thread'] = 0
2950             for content in contents:
2951                 if not parent and tracker['total_parent_comments'] >= max_parents:
2952                     yield
2953                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2954                 comment_renderer = get_first(
2955                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2956                     expected_type=dict, default={})
2957
2958                 comment = self._extract_comment(comment_renderer, parent)
2959                 if not comment:
2960                     continue
2961
2962                 tracker['running_total'] += 1
2963                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2964                 yield comment
2965
2966                 # Attempt to get the replies
2967                 comment_replies_renderer = try_get(
2968                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2969
2970                 if comment_replies_renderer:
2971                     tracker['current_page_thread'] += 1
2972                     comment_entries_iter = self._comment_entries(
2973                         comment_replies_renderer, ytcfg, video_id,
2974                         parent=comment.get('id'), tracker=tracker)
2975                     yield from itertools.islice(comment_entries_iter, min(
2976                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2977
2978         # Keeps track of counts across recursive calls
2979         if not tracker:
2980             tracker = dict(
2981                 running_total=0,
2982                 est_total=0,
2983                 current_page_thread=0,
2984                 total_parent_comments=0,
2985                 total_reply_comments=0)
2986
2987         # TODO: Deprecated
2988         # YouTube comments have a max depth of 2
2989         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2990         if max_depth:
2991             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
2992                                                 'Set max replies in the max-comments extractor argument instead')
2993         if max_depth == 1 and parent:
2994             return
2995
2996         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2997             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2998
2999         continuation = self._extract_continuation(root_continuation_data)
3000
3001         response = None
3002         is_forced_continuation = False
3003         is_first_continuation = parent is None
3004         if is_first_continuation and not continuation:
3005             # Sometimes you can get comments by generating the continuation yourself,
3006             # even if YouTube initially reports them being disabled - e.g. stories comments.
3007             # Note: if the comment section is actually disabled, YouTube may return a response with
3008             # required check_get_keys missing. So we will disable that check initially in this case.
3009             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3010             is_forced_continuation = True
3011
3012         for page_num in itertools.count(0):
3013             if not continuation:
3014                 break
3015             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3016             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3017             if page_num == 0:
3018                 if is_first_continuation:
3019                     note_prefix = 'Downloading comment section API JSON'
3020                 else:
3021                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3022                         tracker['current_page_thread'], comment_prog_str)
3023             else:
3024                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3025                     '       ' if parent else '', ' replies' if parent else '',
3026                     page_num, comment_prog_str)
3027
3028             response = self._extract_response(
3029                 item_id=None, query=continuation,
3030                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3031                 check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3032             is_forced_continuation = False
3033             continuation_contents = traverse_obj(
3034                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3035
3036             continuation = None
3037             for continuation_section in continuation_contents:
3038                 continuation_items = traverse_obj(
3039                     continuation_section,
3040                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3041                     get_all=False, expected_type=list) or []
3042                 if is_first_continuation:
3043                     continuation = extract_header(continuation_items)
3044                     is_first_continuation = False
3045                     if continuation:
3046                         break
3047                     continue
3048
3049                 for entry in extract_thread(continuation_items):
3050                     if not entry:
3051                         return
3052                     yield entry
3053                 continuation = self._extract_continuation({'contents': continuation_items})
3054                 if continuation:
3055                     break
3056
3057         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3058         if message and not parent and tracker['running_total'] == 0:
3059             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3060
3061     @staticmethod
3062     def _generate_comment_continuation(video_id):
3063         """
3064         Generates initial comment section continuation token from given video id
3065         """
3066         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3067         return base64.b64encode(token.encode()).decode()
3068
3069     def _get_comments(self, ytcfg, video_id, contents, webpage):
3070         """Entry for comment extraction"""
3071         def _real_comment_extract(contents):
3072             renderer = next((
3073                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3074                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3075             yield from self._comment_entries(renderer, ytcfg, video_id)
3076
3077         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3078         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3079
3080     @staticmethod
3081     def _get_checkok_params():
3082         return {'contentCheckOk': True, 'racyCheckOk': True}
3083
3084     @classmethod
3085     def _generate_player_context(cls, sts=None):
3086         context = {
3087             'html5Preference': 'HTML5_PREF_WANTS',
3088         }
3089         if sts is not None:
3090             context['signatureTimestamp'] = sts
3091         return {
3092             'playbackContext': {
3093                 'contentPlaybackContext': context
3094             },
3095             **cls._get_checkok_params()
3096         }
3097
3098     @staticmethod
3099     def _is_agegated(player_response):
3100         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3101             return True
3102
3103         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
3104         AGE_GATE_REASONS = (
3105             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3106             'age_verification_required', 'age_check_required',  # status
3107         )
3108         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3109
3110     @staticmethod
3111     def _is_unplayable(player_response):
3112         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3113
3114     _STORY_PLAYER_PARAMS = '8AEB'
3115
3116     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3117
3118         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3119         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3120         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3121         headers = self.generate_api_headers(
3122             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3123
3124         yt_query = {
3125             'videoId': video_id,
3126         }
3127         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3128             yt_query['params'] = self._STORY_PLAYER_PARAMS
3129
3130         yt_query.update(self._generate_player_context(sts))
3131         return self._extract_response(
3132             item_id=video_id, ep='player', query=yt_query,
3133             ytcfg=player_ytcfg, headers=headers, fatal=True,
3134             default_client=client,
3135             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3136         ) or None
3137
3138     def _get_requested_clients(self, url, smuggled_data):
3139         requested_clients = []
3140         default = ['android', 'web']
3141         allowed_clients = sorted(
3142             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3143             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3144         for client in self._configuration_arg('player_client'):
3145             if client in allowed_clients:
3146                 requested_clients.append(client)
3147             elif client == 'default':
3148                 requested_clients.extend(default)
3149             elif client == 'all':
3150                 requested_clients.extend(allowed_clients)
3151             else:
3152                 self.report_warning(f'Skipping unsupported client {client}')
3153         if not requested_clients:
3154             requested_clients = default
3155
3156         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3157             requested_clients.extend(
3158                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3159
3160         return orderedSet(requested_clients)
3161
3162     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3163         initial_pr = None
3164         if webpage:
3165             initial_pr = self._search_json(
3166                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3167
3168         all_clients = set(clients)
3169         clients = clients[::-1]
3170         prs = []
3171
3172         def append_client(*client_names):
3173             """ Append the first client name that exists but not already used """
3174             for client_name in client_names:
3175                 actual_client = _split_innertube_client(client_name)[0]
3176                 if actual_client in INNERTUBE_CLIENTS:
3177                     if actual_client not in all_clients:
3178                         clients.append(client_name)
3179                         all_clients.add(actual_client)
3180                         return
3181
3182         # Android player_response does not have microFormats which are needed for
3183         # extraction of some data. So we return the initial_pr with formats
3184         # stripped out even if not requested by the user
3185         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3186         if initial_pr:
3187             pr = dict(initial_pr)
3188             pr['streamingData'] = None
3189             prs.append(pr)
3190
3191         last_error = None
3192         tried_iframe_fallback = False
3193         player_url = None
3194         while clients:
3195             client, base_client, variant = _split_innertube_client(clients.pop())
3196             player_ytcfg = master_ytcfg if client == 'web' else {}
3197             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3198                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3199
3200             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3201             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3202             if 'js' in self._configuration_arg('player_skip'):
3203                 require_js_player = False
3204                 player_url = None
3205
3206             if not player_url and not tried_iframe_fallback and require_js_player:
3207                 player_url = self._download_player_url(video_id)
3208                 tried_iframe_fallback = True
3209
3210             try:
3211                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3212                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3213             except ExtractorError as e:
3214                 if last_error:
3215                     self.report_warning(last_error)
3216                 last_error = e
3217                 continue
3218
3219             if pr:
3220                 # YouTube may return a different video player response than expected.
3221                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3222                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3223                 if pr_video_id and pr_video_id != video_id:
3224                     self.report_warning(
3225                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3226                 else:
3227                     prs.append(pr)
3228
3229             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3230             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3231                 append_client(f'{base_client}_creator')
3232             elif self._is_agegated(pr):
3233                 if variant == 'tv_embedded':
3234                     append_client(f'{base_client}_embedded')
3235                 elif not variant:
3236                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3237
3238         if last_error:
3239             if not len(prs):
3240                 raise last_error
3241             self.report_warning(last_error)
3242         return prs, player_url
3243
3244     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, is_live, duration):
3245         itags, stream_ids = {}, []
3246         itag_qualities, res_qualities = {}, {0: None}
3247         q = qualities([
3248             # Normally tiny is the smallest video-only formats. But
3249             # audio-only formats with unknown quality may get tagged as tiny
3250             'tiny',
3251             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3252             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3253         ])
3254         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3255
3256         for fmt in streaming_formats:
3257             if fmt.get('targetDurationSec'):
3258                 continue
3259
3260             itag = str_or_none(fmt.get('itag'))
3261             audio_track = fmt.get('audioTrack') or {}
3262             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3263             if stream_id in stream_ids:
3264                 continue
3265
3266             quality = fmt.get('quality')
3267             height = int_or_none(fmt.get('height'))
3268             if quality == 'tiny' or not quality:
3269                 quality = fmt.get('audioQuality', '').lower() or quality
3270             # The 3gp format (17) in android client has a quality of "small",
3271             # but is actually worse than other formats
3272             if itag == '17':
3273                 quality = 'tiny'
3274             if quality:
3275                 if itag:
3276                     itag_qualities[itag] = quality
3277                 if height:
3278                     res_qualities[height] = quality
3279             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3280             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3281             # number of fragment that would subsequently requested with (`&sq=N`)
3282             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3283                 continue
3284
3285             fmt_url = fmt.get('url')
3286             if not fmt_url:
3287                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3288                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3289                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3290                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3291                     continue
3292                 try:
3293                     fmt_url += '&%s=%s' % (
3294                         traverse_obj(sc, ('sp', -1)) or 'signature',
3295                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3296                     )
3297                 except ExtractorError as e:
3298                     self.report_warning('Signature extraction failed: Some formats may be missing',
3299                                         video_id=video_id, only_once=True)
3300                     self.write_debug(e, only_once=True)
3301                     continue
3302
3303             query = parse_qs(fmt_url)
3304             throttled = False
3305             if query.get('n'):
3306                 try:
3307                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3308                     fmt_url = update_url_query(fmt_url, {
3309                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3310                     })
3311                 except ExtractorError as e:
3312                     phantomjs_hint = ''
3313                     if isinstance(e, JSInterpreter.Exception):
3314                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3315                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3316                     self.report_warning(
3317                         f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3318                         f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3319                     self.write_debug(e, only_once=True)
3320                     throttled = True
3321
3322             if itag:
3323                 itags[itag] = 'https'
3324                 stream_ids.append(stream_id)
3325
3326             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3327             language_preference = (
3328                 10 if audio_track.get('audioIsDefault') and 10
3329                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3330                 else -1)
3331             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3332             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3333             # Make sure to avoid false positives with small duration differences.
3334             # E.g. __2ABJjxzNo, ySuUZEjARPY
3335             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3336             if is_damaged:
3337                 self.report_warning(
3338                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3339             dct = {
3340                 'asr': int_or_none(fmt.get('audioSampleRate')),
3341                 'filesize': int_or_none(fmt.get('contentLength')),
3342                 'format_id': itag,
3343                 'format_note': join_nonempty(
3344                     '%s%s' % (audio_track.get('displayName') or '',
3345                               ' (default)' if language_preference > 0 else ''),
3346                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3347                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3348                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3349                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3350                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3351                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3352                 'fps': int_or_none(fmt.get('fps')) or None,
3353                 'audio_channels': fmt.get('audioChannels'),
3354                 'height': height,
3355                 'quality': q(quality),
3356                 'has_drm': bool(fmt.get('drmFamilies')),
3357                 'tbr': tbr,
3358                 'url': fmt_url,
3359                 'width': int_or_none(fmt.get('width')),
3360                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3361                                           'desc' if language_preference < -1 else ''),
3362                 'language_preference': language_preference,
3363                 # Strictly de-prioritize damaged and 3gp formats
3364                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3365             }
3366             mime_mobj = re.match(
3367                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3368             if mime_mobj:
3369                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3370                 dct.update(parse_codecs(mime_mobj.group(2)))
3371             no_audio = dct.get('acodec') == 'none'
3372             no_video = dct.get('vcodec') == 'none'
3373             if no_audio:
3374                 dct['vbr'] = tbr
3375             if no_video:
3376                 dct['abr'] = tbr
3377             if no_audio or no_video:
3378                 dct['downloader_options'] = {
3379                     # Youtube throttles chunks >~10M
3380                     'http_chunk_size': 10485760,
3381                 }
3382                 if dct.get('ext'):
3383                     dct['container'] = dct['ext'] + '_dash'
3384             yield dct
3385
3386         live_from_start = is_live and self.get_param('live_from_start')
3387         skip_manifests = self._configuration_arg('skip')
3388         if not self.get_param('youtube_include_hls_manifest', True):
3389             skip_manifests.append('hls')
3390         if not self.get_param('youtube_include_dash_manifest', True):
3391             skip_manifests.append('dash')
3392         get_dash = 'dash' not in skip_manifests and (
3393             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3394         get_hls = not live_from_start and 'hls' not in skip_manifests
3395
3396         def process_manifest_format(f, proto, itag):
3397             if itag in itags:
3398                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3399                     return False
3400                 itag = f'{itag}-{proto}'
3401             if itag:
3402                 f['format_id'] = itag
3403                 itags[itag] = proto
3404
3405             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3406             if f['quality'] == -1 and f.get('height'):
3407                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3408             return True
3409
3410         subtitles = {}
3411         for sd in streaming_data:
3412             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3413             if hls_manifest_url:
3414                 fmts, subs = self._extract_m3u8_formats_and_subtitles(hls_manifest_url, video_id, 'mp4', fatal=False, live=is_live)
3415                 subtitles = self._merge_subtitles(subs, subtitles)
3416                 for f in fmts:
3417                     if process_manifest_format(f, 'hls', self._search_regex(
3418                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3419                         yield f
3420
3421             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3422             if dash_manifest_url:
3423                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3424                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3425                 for f in formats:
3426                     if process_manifest_format(f, 'dash', f['format_id']):
3427                         f['filesize'] = int_or_none(self._search_regex(
3428                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3429                         if live_from_start:
3430                             f['is_from_start'] = True
3431
3432                         yield f
3433         yield subtitles
3434
3435     def _extract_storyboard(self, player_responses, duration):
3436         spec = get_first(
3437             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3438         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3439         if not base_url:
3440             return
3441         L = len(spec) - 1
3442         for i, args in enumerate(spec):
3443             args = args.split('#')
3444             counts = list(map(int_or_none, args[:5]))
3445             if len(args) != 8 or not all(counts):
3446                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3447                 continue
3448             width, height, frame_count, cols, rows = counts
3449             N, sigh = args[6:]
3450
3451             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3452             fragment_count = frame_count / (cols * rows)
3453             fragment_duration = duration / fragment_count
3454             yield {
3455                 'format_id': f'sb{i}',
3456                 'format_note': 'storyboard',
3457                 'ext': 'mhtml',
3458                 'protocol': 'mhtml',
3459                 'acodec': 'none',
3460                 'vcodec': 'none',
3461                 'url': url,
3462                 'width': width,
3463                 'height': height,
3464                 'fps': frame_count / duration,
3465                 'rows': rows,
3466                 'columns': cols,
3467                 'fragments': [{
3468                     'url': url.replace('$M', str(j)),
3469                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3470                 } for j in range(math.ceil(fragment_count))],
3471             }
3472
3473     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3474         webpage = None
3475         if 'webpage' not in self._configuration_arg('player_skip'):
3476             query = {'bpctr': '9999999999', 'has_verified': '1'}
3477             if smuggled_data.get('is_story'):
3478                 query['pp'] = self._STORY_PLAYER_PARAMS
3479             webpage = self._download_webpage(
3480                 webpage_url, video_id, fatal=False, query=query)
3481
3482         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3483
3484         player_responses, player_url = self._extract_player_responses(
3485             self._get_requested_clients(url, smuggled_data),
3486             video_id, webpage, master_ytcfg, smuggled_data)
3487
3488         return webpage, master_ytcfg, player_responses, player_url
3489
3490     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3491         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3492         is_live = get_first(video_details, 'isLive')
3493         if is_live is None:
3494             is_live = get_first(live_broadcast_details, 'isLiveNow')
3495
3496         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3497         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, is_live, duration)
3498
3499         return live_broadcast_details, is_live, streaming_data, formats, subtitles
3500
3501     def _real_extract(self, url):
3502         url, smuggled_data = unsmuggle_url(url, {})
3503         video_id = self._match_id(url)
3504
3505         base_url = self.http_scheme() + '//www.youtube.com/'
3506         webpage_url = base_url + 'watch?v=' + video_id
3507
3508         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3509
3510         playability_statuses = traverse_obj(
3511             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3512
3513         trailer_video_id = get_first(
3514             playability_statuses,
3515             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3516             expected_type=str)
3517         if trailer_video_id:
3518             return self.url_result(
3519                 trailer_video_id, self.ie_key(), trailer_video_id)
3520
3521         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3522                        if webpage else (lambda x: None))
3523
3524         video_details = traverse_obj(
3525             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3526         microformats = traverse_obj(
3527             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3528             expected_type=dict, default=[])
3529         video_title = (
3530             get_first(video_details, 'title')
3531             or self._get_text(microformats, (..., 'title'))
3532             or search_meta(['og:title', 'twitter:title', 'title']))
3533         video_description = get_first(video_details, 'shortDescription')
3534
3535         multifeed_metadata_list = get_first(
3536             player_responses,
3537             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3538             expected_type=str)
3539         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3540             if self.get_param('noplaylist'):
3541                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3542             else:
3543                 entries = []
3544                 feed_ids = []
3545                 for feed in multifeed_metadata_list.split(','):
3546                     # Unquote should take place before split on comma (,) since textual
3547                     # fields may contain comma as well (see
3548                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3549                     feed_data = urllib.parse.parse_qs(
3550                         urllib.parse.unquote_plus(feed))
3551
3552                     def feed_entry(name):
3553                         return try_get(
3554                             feed_data, lambda x: x[name][0], str)
3555
3556                     feed_id = feed_entry('id')
3557                     if not feed_id:
3558                         continue
3559                     feed_title = feed_entry('title')
3560                     title = video_title
3561                     if feed_title:
3562                         title += ' (%s)' % feed_title
3563                     entries.append({
3564                         '_type': 'url_transparent',
3565                         'ie_key': 'Youtube',
3566                         'url': smuggle_url(
3567                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3568                             {'force_singlefeed': True}),
3569                         'title': title,
3570                     })
3571                     feed_ids.append(feed_id)
3572                 self.to_screen(
3573                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3574                     % (', '.join(feed_ids), video_id))
3575                 return self.playlist_result(
3576                     entries, video_id, video_title, video_description)
3577
3578         duration = int_or_none(
3579             get_first(video_details, 'lengthSeconds')
3580             or get_first(microformats, 'lengthSeconds')
3581             or parse_duration(search_meta('duration'))) or None
3582
3583         live_broadcast_details, is_live, streaming_data, formats, automatic_captions = \
3584             self._list_formats(video_id, microformats, video_details, player_responses, player_url)
3585
3586         if not formats:
3587             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3588                 self.report_drm(video_id)
3589             pemr = get_first(
3590                 playability_statuses,
3591                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3592             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3593             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3594             if subreason:
3595                 if subreason == 'The uploader has not made this video available in your country.':
3596                     countries = get_first(microformats, 'availableCountries')
3597                     if not countries:
3598                         regions_allowed = search_meta('regionsAllowed')
3599                         countries = regions_allowed.split(',') if regions_allowed else None
3600                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3601                 reason += f'. {subreason}'
3602             if reason:
3603                 self.raise_no_formats(reason, expected=True)
3604
3605         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3606         if not keywords and webpage:
3607             keywords = [
3608                 unescapeHTML(m.group('content'))
3609                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3610         for keyword in keywords:
3611             if keyword.startswith('yt:stretch='):
3612                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3613                 if mobj:
3614                     # NB: float is intentional for forcing float division
3615                     w, h = (float(v) for v in mobj.groups())
3616                     if w > 0 and h > 0:
3617                         ratio = w / h
3618                         for f in formats:
3619                             if f.get('vcodec') != 'none':
3620                                 f['stretched_ratio'] = ratio
3621                         break
3622         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3623         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3624         if thumbnail_url:
3625             thumbnails.append({
3626                 'url': thumbnail_url,
3627             })
3628         original_thumbnails = thumbnails.copy()
3629
3630         # The best resolution thumbnails sometimes does not appear in the webpage
3631         # See: https://github.com/yt-dlp/yt-dlp/issues/340
3632         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3633         thumbnail_names = [
3634             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
3635             # in resolution, these are not the custom thumbnail. So de-prioritize them
3636             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
3637             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
3638         ]
3639         n_thumbnail_names = len(thumbnail_names)
3640         thumbnails.extend({
3641             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3642                 video_id=video_id, name=name, ext=ext,
3643                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3644         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3645         for thumb in thumbnails:
3646             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3647             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3648         self._remove_duplicate_formats(thumbnails)
3649         self._downloader._sort_thumbnails(original_thumbnails)
3650
3651         category = get_first(microformats, 'category') or search_meta('genre')
3652         channel_id = str_or_none(
3653             get_first(video_details, 'channelId')
3654             or get_first(microformats, 'externalChannelId')
3655             or search_meta('channelId'))
3656         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3657
3658         live_content = get_first(video_details, 'isLiveContent')
3659         is_upcoming = get_first(video_details, 'isUpcoming')
3660         if is_live is None:
3661             if is_upcoming or live_content is False:
3662                 is_live = False
3663         if is_upcoming is None and (live_content or is_live):
3664             is_upcoming = False
3665         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3666         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3667         if not duration and live_end_time and live_start_time:
3668             duration = live_end_time - live_start_time
3669
3670         if is_live and self.get_param('live_from_start'):
3671             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3672
3673         formats.extend(self._extract_storyboard(player_responses, duration))
3674
3675         # source_preference is lower for throttled/potentially damaged formats
3676         self._sort_formats(formats, (
3677             'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto'))
3678
3679         info = {
3680             'id': video_id,
3681             'title': video_title,
3682             'formats': formats,
3683             'thumbnails': thumbnails,
3684             # The best thumbnail that we are sure exists. Prevents unnecessary
3685             # URL checking if user don't care about getting the best possible thumbnail
3686             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3687             'description': video_description,
3688             'uploader': get_first(video_details, 'author'),
3689             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3690             'uploader_url': owner_profile_url,
3691             'channel_id': channel_id,
3692             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
3693             'duration': duration,
3694             'view_count': int_or_none(
3695                 get_first((video_details, microformats), (..., 'viewCount'))
3696                 or search_meta('interactionCount')),
3697             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3698             'age_limit': 18 if (
3699                 get_first(microformats, 'isFamilySafe') is False
3700                 or search_meta('isFamilyFriendly') == 'false'
3701                 or search_meta('og:restrictions:age') == '18+') else 0,
3702             'webpage_url': webpage_url,
3703             'categories': [category] if category else None,
3704             'tags': keywords,
3705             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3706             'is_live': is_live,
3707             'was_live': (False if is_live or is_upcoming or live_content is False
3708                          else None if is_live is None or is_upcoming is None
3709                          else live_content),
3710             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3711             'release_timestamp': live_start_time,
3712         }
3713
3714         if get_first(video_details, 'isPostLiveDvr'):
3715             self.write_debug('Video is in Post-Live Manifestless mode')
3716             info['live_status'] = 'post_live'
3717             if (duration or 0) > 4 * 3600:
3718                 self.report_warning(
3719                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
3720                     'This is a known issue and patches are welcome')
3721
3722         subtitles = {}
3723         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3724         if pctr:
3725             def get_lang_code(track):
3726                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3727                         or track.get('languageCode'))
3728
3729             # Converted into dicts to remove duplicates
3730             captions = {
3731                 get_lang_code(sub): sub
3732                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3733             translation_languages = {
3734                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3735                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3736
3737             def process_language(container, base_url, lang_code, sub_name, query):
3738                 lang_subs = container.setdefault(lang_code, [])
3739                 for fmt in self._SUBTITLE_FORMATS:
3740                     query.update({
3741                         'fmt': fmt,
3742                     })
3743                     lang_subs.append({
3744                         'ext': fmt,
3745                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3746                         'name': sub_name,
3747                     })
3748
3749             # NB: Constructing the full subtitle dictionary is slow
3750             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
3751                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
3752             for lang_code, caption_track in captions.items():
3753                 base_url = caption_track.get('baseUrl')
3754                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3755                 if not base_url:
3756                     continue
3757                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3758                 if caption_track.get('kind') != 'asr':
3759                     if not lang_code:
3760                         continue
3761                     process_language(
3762                         subtitles, base_url, lang_code, lang_name, {})
3763                     if not caption_track.get('isTranslatable'):
3764                         continue
3765                 for trans_code, trans_name in translation_languages.items():
3766                     if not trans_code:
3767                         continue
3768                     orig_trans_code = trans_code
3769                     if caption_track.get('kind') != 'asr':
3770                         if not get_translated_subs:
3771                             continue
3772                         trans_code += f'-{lang_code}'
3773                         trans_name += format_field(lang_name, None, ' from %s')
3774                     # Add an "-orig" label to the original language so that it can be distinguished.
3775                     # The subs are returned without "-orig" as well for compatibility
3776                     if lang_code == f'a-{orig_trans_code}':
3777                         process_language(
3778                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3779                     # Setting tlang=lang returns damaged subtitles.
3780                     process_language(automatic_captions, base_url, trans_code, trans_name,
3781                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3782
3783         info['automatic_captions'] = automatic_captions
3784         info['subtitles'] = subtitles
3785
3786         parsed_url = urllib.parse.urlparse(url)
3787         for component in [parsed_url.fragment, parsed_url.query]:
3788             query = urllib.parse.parse_qs(component)
3789             for k, v in query.items():
3790                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3791                     d_k += '_time'
3792                     if d_k not in info and k in s_ks:
3793                         info[d_k] = parse_duration(query[k][0])
3794
3795         # Youtube Music Auto-generated description
3796         if video_description:
3797             mobj = re.search(
3798                 r'''(?xs)
3799                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
3800                     (?P<album>[^\n]+)
3801                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
3802                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
3803                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
3804                     .+\nAuto-generated\ by\ YouTube\.\s*$
3805                 ''', video_description)
3806             if mobj:
3807                 release_year = mobj.group('release_year')
3808                 release_date = mobj.group('release_date')
3809                 if release_date:
3810                     release_date = release_date.replace('-', '')
3811                     if not release_year:
3812                         release_year = release_date[:4]
3813                 info.update({
3814                     'album': mobj.group('album'.strip()),
3815                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3816                     'track': mobj.group('track').strip(),
3817                     'release_date': release_date,
3818                     'release_year': int_or_none(release_year),
3819                 })
3820
3821         initial_data = None
3822         if webpage:
3823             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
3824         if not initial_data:
3825             query = {'videoId': video_id}
3826             query.update(self._get_checkok_params())
3827             initial_data = self._extract_response(
3828                 item_id=video_id, ep='next', fatal=False,
3829                 ytcfg=master_ytcfg, query=query,
3830                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3831                 note='Downloading initial data API JSON')
3832
3833         info['comment_count'] = traverse_obj(initial_data, (
3834             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
3835             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
3836         ), (
3837             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
3838             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
3839         ), expected_type=int_or_none, get_all=False)
3840
3841         try:  # This will error if there is no livechat
3842             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3843         except (KeyError, IndexError, TypeError):
3844             pass
3845         else:
3846             info.setdefault('subtitles', {})['live_chat'] = [{
3847                 # url is needed to set cookies
3848                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
3849                 'video_id': video_id,
3850                 'ext': 'json',
3851                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3852             }]
3853
3854         if initial_data:
3855             info['chapters'] = (
3856                 self._extract_chapters_from_json(initial_data, duration)
3857                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3858                 or self._extract_chapters_from_description(video_description, duration)
3859                 or None)
3860
3861         contents = traverse_obj(
3862             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3863             expected_type=list, default=[])
3864
3865         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3866         if vpir:
3867             stl = vpir.get('superTitleLink')
3868             if stl:
3869                 stl = self._get_text(stl)
3870                 if try_get(
3871                         vpir,
3872                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3873                     info['location'] = stl
3874                 else:
3875                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
3876                     if mobj:
3877                         info.update({
3878                             'series': mobj.group(1),
3879                             'season_number': int(mobj.group(2)),
3880                             'episode_number': int(mobj.group(3)),
3881                         })
3882             for tlb in (try_get(
3883                     vpir,
3884                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3885                     list) or []):
3886                 tbr = tlb.get('toggleButtonRenderer') or {}
3887                 for getter, regex in [(
3888                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3889                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3890                             lambda x: x['accessibility'],
3891                             lambda x: x['accessibilityData']['accessibilityData'],
3892                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3893                     label = (try_get(tbr, getter, dict) or {}).get('label')
3894                     if label:
3895                         mobj = re.match(regex, label)
3896                         if mobj:
3897                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3898                             break
3899             sbr_tooltip = try_get(
3900                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3901             if sbr_tooltip:
3902                 like_count, dislike_count = sbr_tooltip.split(' / ')
3903                 info.update({
3904                     'like_count': str_to_int(like_count),
3905                     'dislike_count': str_to_int(dislike_count),
3906                 })
3907         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3908         if vsir:
3909             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3910             info.update({
3911                 'channel': self._get_text(vor, 'title'),
3912                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3913
3914             rows = try_get(
3915                 vsir,
3916                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3917                 list) or []
3918             multiple_songs = False
3919             for row in rows:
3920                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3921                     multiple_songs = True
3922                     break
3923             for row in rows:
3924                 mrr = row.get('metadataRowRenderer') or {}
3925                 mrr_title = mrr.get('title')
3926                 if not mrr_title:
3927                     continue
3928                 mrr_title = self._get_text(mrr, 'title')
3929                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3930                 if mrr_title == 'License':
3931                     info['license'] = mrr_contents_text
3932                 elif not multiple_songs:
3933                     if mrr_title == 'Album':
3934                         info['album'] = mrr_contents_text
3935                     elif mrr_title == 'Artist':
3936                         info['artist'] = mrr_contents_text
3937                     elif mrr_title == 'Song':
3938                         info['track'] = mrr_contents_text
3939
3940         fallbacks = {
3941             'channel': 'uploader',
3942             'channel_id': 'uploader_id',
3943             'channel_url': 'uploader_url',
3944         }
3945
3946         # The upload date for scheduled, live and past live streams / premieres in microformats
3947         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3948         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3949         upload_date = (
3950             unified_strdate(get_first(microformats, 'uploadDate'))
3951             or unified_strdate(search_meta('uploadDate')))
3952         if not upload_date or (
3953             not info.get('is_live')
3954             and not info.get('was_live')
3955             and info.get('live_status') != 'is_upcoming'
3956             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
3957         ):
3958             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d') or upload_date
3959         info['upload_date'] = upload_date
3960
3961         for to, frm in fallbacks.items():
3962             if not info.get(to):
3963                 info[to] = info.get(frm)
3964
3965         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3966             v = info.get(s_k)
3967             if v:
3968                 info[d_k] = v
3969
3970         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3971         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3972         is_membersonly = None
3973         is_premium = None
3974         if initial_data and is_private is not None:
3975             is_membersonly = False
3976             is_premium = False
3977             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3978             badge_labels = set()
3979             for content in contents:
3980                 if not isinstance(content, dict):
3981                     continue
3982                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3983             for badge_label in badge_labels:
3984                 if badge_label.lower() == 'members only':
3985                     is_membersonly = True
3986                 elif badge_label.lower() == 'premium':
3987                     is_premium = True
3988                 elif badge_label.lower() == 'unlisted':
3989                     is_unlisted = True
3990
3991         info['availability'] = self._availability(
3992             is_private=is_private,
3993             needs_premium=is_premium,
3994             needs_subscription=is_membersonly,
3995             needs_auth=info['age_limit'] >= 18,
3996             is_unlisted=None if is_private is None else is_unlisted)
3997
3998         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3999
4000         self.mark_watched(video_id, player_responses)
4001
4002         return info
4003
4004
4005 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4006
4007     @staticmethod
4008     def passthrough_smuggled_data(func):
4009         def _smuggle(entries, smuggled_data):
4010             for entry in entries:
4011                 # TODO: Convert URL to music.youtube instead.
4012                 # Do we need to passthrough any other smuggled_data?
4013                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
4014                 yield entry
4015
4016         @functools.wraps(func)
4017         def wrapper(self, url):
4018             url, smuggled_data = unsmuggle_url(url, {})
4019             if self.is_music_url(url):
4020                 smuggled_data['is_music_url'] = True
4021             info_dict = func(self, url, smuggled_data)
4022             if smuggled_data and info_dict.get('entries'):
4023                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
4024             return info_dict
4025         return wrapper
4026
4027     def _extract_channel_id(self, webpage):
4028         channel_id = self._html_search_meta(
4029             'channelId', webpage, 'channel id', default=None)
4030         if channel_id:
4031             return channel_id
4032         channel_url = self._html_search_meta(
4033             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
4034              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
4035              'twitter:app:url:googleplay'), webpage, 'channel url')
4036         return self._search_regex(
4037             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
4038             channel_url, 'channel id')
4039
4040     @staticmethod
4041     def _extract_basic_item_renderer(item):
4042         # Modified from _extract_grid_item_renderer
4043         known_basic_renderers = (
4044             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4045         )
4046         for key, renderer in item.items():
4047             if not isinstance(renderer, dict):
4048                 continue
4049             elif key in known_basic_renderers:
4050                 return renderer
4051             elif key.startswith('grid') and key.endswith('Renderer'):
4052                 return renderer
4053
4054     def _grid_entries(self, grid_renderer):
4055         for item in grid_renderer['items']:
4056             if not isinstance(item, dict):
4057                 continue
4058             renderer = self._extract_basic_item_renderer(item)
4059             if not isinstance(renderer, dict):
4060                 continue
4061             title = self._get_text(renderer, 'title')
4062
4063             # playlist
4064             playlist_id = renderer.get('playlistId')
4065             if playlist_id:
4066                 yield self.url_result(
4067                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4068                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4069                     video_title=title)
4070                 continue
4071             # video
4072             video_id = renderer.get('videoId')
4073             if video_id:
4074                 yield self._extract_video(renderer)
4075                 continue
4076             # channel
4077             channel_id = renderer.get('channelId')
4078             if channel_id:
4079                 yield self.url_result(
4080                     'https://www.youtube.com/channel/%s' % channel_id,
4081                     ie=YoutubeTabIE.ie_key(), video_title=title)
4082                 continue
4083             # generic endpoint URL support
4084             ep_url = urljoin('https://www.youtube.com/', try_get(
4085                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4086                 str))
4087             if ep_url:
4088                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4089                     if ie.suitable(ep_url):
4090                         yield self.url_result(
4091                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4092                         break
4093
4094     def _music_reponsive_list_entry(self, renderer):
4095         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4096         if video_id:
4097             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4098                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4099         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4100         if playlist_id:
4101             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4102             if video_id:
4103                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4104                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4105             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4106                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4107         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4108         if browse_id:
4109             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4110                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4111
4112     def _shelf_entries_from_content(self, shelf_renderer):
4113         content = shelf_renderer.get('content')
4114         if not isinstance(content, dict):
4115             return
4116         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4117         if renderer:
4118             # TODO: add support for nested playlists so each shelf is processed
4119             # as separate playlist
4120             # TODO: this includes only first N items
4121             yield from self._grid_entries(renderer)
4122         renderer = content.get('horizontalListRenderer')
4123         if renderer:
4124             # TODO
4125             pass
4126
4127     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4128         ep = try_get(
4129             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4130             str)
4131         shelf_url = urljoin('https://www.youtube.com', ep)
4132         if shelf_url:
4133             # Skipping links to another channels, note that checking for
4134             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4135             # will not work
4136             if skip_channels and '/channels?' in shelf_url:
4137                 return
4138             title = self._get_text(shelf_renderer, 'title')
4139             yield self.url_result(shelf_url, video_title=title)
4140         # Shelf may not contain shelf URL, fallback to extraction from content
4141         yield from self._shelf_entries_from_content(shelf_renderer)
4142
4143     def _playlist_entries(self, video_list_renderer):
4144         for content in video_list_renderer['contents']:
4145             if not isinstance(content, dict):
4146                 continue
4147             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4148             if not isinstance(renderer, dict):
4149                 continue
4150             video_id = renderer.get('videoId')
4151             if not video_id:
4152                 continue
4153             yield self._extract_video(renderer)
4154
4155     def _rich_entries(self, rich_grid_renderer):
4156         renderer = try_get(
4157             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
4158         video_id = renderer.get('videoId')
4159         if not video_id:
4160             return
4161         yield self._extract_video(renderer)
4162
4163     def _video_entry(self, video_renderer):
4164         video_id = video_renderer.get('videoId')
4165         if video_id:
4166             return self._extract_video(video_renderer)
4167
4168     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4169         url = urljoin('https://youtube.com', traverse_obj(
4170             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4171         if url:
4172             return self.url_result(
4173                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4174
4175     def _post_thread_entries(self, post_thread_renderer):
4176         post_renderer = try_get(
4177             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4178         if not post_renderer:
4179             return
4180         # video attachment
4181         video_renderer = try_get(
4182             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4183         video_id = video_renderer.get('videoId')
4184         if video_id:
4185             entry = self._extract_video(video_renderer)
4186             if entry:
4187                 yield entry
4188         # playlist attachment
4189         playlist_id = try_get(
4190             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4191         if playlist_id:
4192             yield self.url_result(
4193                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4194                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4195         # inline video links
4196         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4197         for run in runs:
4198             if not isinstance(run, dict):
4199                 continue
4200             ep_url = try_get(
4201                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4202             if not ep_url:
4203                 continue
4204             if not YoutubeIE.suitable(ep_url):
4205                 continue
4206             ep_video_id = YoutubeIE._match_id(ep_url)
4207             if video_id == ep_video_id:
4208                 continue
4209             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4210
4211     def _post_thread_continuation_entries(self, post_thread_continuation):
4212         contents = post_thread_continuation.get('contents')
4213         if not isinstance(contents, list):
4214             return
4215         for content in contents:
4216             renderer = content.get('backstagePostThreadRenderer')
4217             if isinstance(renderer, dict):
4218                 yield from self._post_thread_entries(renderer)
4219                 continue
4220             renderer = content.get('videoRenderer')
4221             if isinstance(renderer, dict):
4222                 yield self._video_entry(renderer)
4223
4224     r''' # unused
4225     def _rich_grid_entries(self, contents):
4226         for content in contents:
4227             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4228             if video_renderer:
4229                 entry = self._video_entry(video_renderer)
4230                 if entry:
4231                     yield entry
4232     '''
4233
4234     def _extract_entries(self, parent_renderer, continuation_list):
4235         # continuation_list is modified in-place with continuation_list = [continuation_token]
4236         continuation_list[:] = [None]
4237         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4238         for content in contents:
4239             if not isinstance(content, dict):
4240                 continue
4241             is_renderer = traverse_obj(
4242                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4243                 expected_type=dict)
4244             if not is_renderer:
4245                 renderer = content.get('richItemRenderer')
4246                 if renderer:
4247                     for entry in self._rich_entries(renderer):
4248                         yield entry
4249                     continuation_list[0] = self._extract_continuation(parent_renderer)
4250                 continue
4251             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4252             for isr_content in isr_contents:
4253                 if not isinstance(isr_content, dict):
4254                     continue
4255
4256                 known_renderers = {
4257                     'playlistVideoListRenderer': self._playlist_entries,
4258                     'gridRenderer': self._grid_entries,
4259                     'reelShelfRenderer': self._grid_entries,
4260                     'shelfRenderer': self._shelf_entries,
4261                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4262                     'backstagePostThreadRenderer': self._post_thread_entries,
4263                     'videoRenderer': lambda x: [self._video_entry(x)],
4264                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4265                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4266                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4267                 }
4268                 for key, renderer in isr_content.items():
4269                     if key not in known_renderers:
4270                         continue
4271                     for entry in known_renderers[key](renderer):
4272                         if entry:
4273                             yield entry
4274                     continuation_list[0] = self._extract_continuation(renderer)
4275                     break
4276
4277             if not continuation_list[0]:
4278                 continuation_list[0] = self._extract_continuation(is_renderer)
4279
4280         if not continuation_list[0]:
4281             continuation_list[0] = self._extract_continuation(parent_renderer)
4282
4283     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4284         continuation_list = [None]
4285         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4286         tab_content = try_get(tab, lambda x: x['content'], dict)
4287         if not tab_content:
4288             return
4289         parent_renderer = (
4290             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4291             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4292         yield from extract_entries(parent_renderer)
4293         continuation = continuation_list[0]
4294
4295         for page_num in itertools.count(1):
4296             if not continuation:
4297                 break
4298             headers = self.generate_api_headers(
4299                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4300             response = self._extract_response(
4301                 item_id=f'{item_id} page {page_num}',
4302                 query=continuation, headers=headers, ytcfg=ytcfg,
4303                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4304
4305             if not response:
4306                 break
4307             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4308             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4309             visitor_data = self._extract_visitor_data(response) or visitor_data
4310
4311             known_continuation_renderers = {
4312                 'playlistVideoListContinuation': self._playlist_entries,
4313                 'gridContinuation': self._grid_entries,
4314                 'itemSectionContinuation': self._post_thread_continuation_entries,
4315                 'sectionListContinuation': extract_entries,  # for feeds
4316             }
4317             continuation_contents = try_get(
4318                 response, lambda x: x['continuationContents'], dict) or {}
4319             continuation_renderer = None
4320             for key, value in continuation_contents.items():
4321                 if key not in known_continuation_renderers:
4322                     continue
4323                 continuation_renderer = value
4324                 continuation_list = [None]
4325                 yield from known_continuation_renderers[key](continuation_renderer)
4326                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4327                 break
4328             if continuation_renderer:
4329                 continue
4330
4331             known_renderers = {
4332                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4333                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4334                 'gridVideoRenderer': (self._grid_entries, 'items'),
4335                 'gridChannelRenderer': (self._grid_entries, 'items'),
4336                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4337                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4338                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4339                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4340             }
4341             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4342             continuation_items = try_get(
4343                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4344             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4345             video_items_renderer = None
4346             for key, value in continuation_item.items():
4347                 if key not in known_renderers:
4348                     continue
4349                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4350                 continuation_list = [None]
4351                 yield from known_renderers[key][0](video_items_renderer)
4352                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4353                 break
4354             if video_items_renderer:
4355                 continue
4356             break
4357
4358     @staticmethod
4359     def _extract_selected_tab(tabs, fatal=True):
4360         for tab in tabs:
4361             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4362             if renderer.get('selected') is True:
4363                 return renderer
4364         else:
4365             if fatal:
4366                 raise ExtractorError('Unable to find selected tab')
4367
4368     def _extract_uploader(self, data):
4369         uploader = {}
4370         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4371         owner = try_get(
4372             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4373         if owner:
4374             owner_text = owner.get('text')
4375             uploader['uploader'] = self._search_regex(
4376                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4377             uploader['uploader_id'] = try_get(
4378                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], str)
4379             uploader['uploader_url'] = urljoin(
4380                 'https://www.youtube.com/',
4381                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], str))
4382         return {k: v for k, v in uploader.items() if v is not None}
4383
4384     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4385         playlist_id = title = description = channel_url = channel_name = channel_id = None
4386         tags = []
4387
4388         selected_tab = self._extract_selected_tab(tabs)
4389         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4390         renderer = try_get(
4391             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4392         if renderer:
4393             channel_name = renderer.get('title')
4394             channel_url = renderer.get('channelUrl')
4395             channel_id = renderer.get('externalId')
4396         else:
4397             renderer = try_get(
4398                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4399
4400         if renderer:
4401             title = renderer.get('title')
4402             description = renderer.get('description', '')
4403             playlist_id = channel_id
4404             tags = renderer.get('keywords', '').split()
4405
4406         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4407         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4408         def _get_uncropped(url):
4409             return url_or_none((url or '').split('=')[0] + '=s0')
4410
4411         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4412         if avatar_thumbnails:
4413             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4414             if uncropped_avatar:
4415                 avatar_thumbnails.append({
4416                     'url': uncropped_avatar,
4417                     'id': 'avatar_uncropped',
4418                     'preference': 1
4419                 })
4420
4421         channel_banners = self._extract_thumbnails(
4422             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4423         for banner in channel_banners:
4424             banner['preference'] = -10
4425
4426         if channel_banners:
4427             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4428             if uncropped_banner:
4429                 channel_banners.append({
4430                     'url': uncropped_banner,
4431                     'id': 'banner_uncropped',
4432                     'preference': -5
4433                 })
4434
4435         primary_thumbnails = self._extract_thumbnails(
4436             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4437
4438         if playlist_id is None:
4439             playlist_id = item_id
4440
4441         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4442         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4443         if title is None:
4444             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4445         title += format_field(selected_tab, 'title', ' - %s')
4446         title += format_field(selected_tab, 'expandedText', ' - %s')
4447
4448         metadata = {
4449             'playlist_id': playlist_id,
4450             'playlist_title': title,
4451             'playlist_description': description,
4452             'uploader': channel_name,
4453             'uploader_id': channel_id,
4454             'uploader_url': channel_url,
4455             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4456             'tags': tags,
4457             'view_count': self._get_count(playlist_stats, 1),
4458             'availability': self._extract_availability(data),
4459             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4460             'playlist_count': self._get_count(playlist_stats, 0),
4461             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4462         }
4463         if not channel_id:
4464             metadata.update(self._extract_uploader(data))
4465         metadata.update({
4466             'channel': metadata['uploader'],
4467             'channel_id': metadata['uploader_id'],
4468             'channel_url': metadata['uploader_url']})
4469         return self.playlist_result(
4470             self._entries(
4471                 selected_tab, playlist_id, ytcfg,
4472                 self._extract_account_syncid(ytcfg, data),
4473                 self._extract_visitor_data(data, ytcfg)),
4474             **metadata)
4475
4476     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4477         first_id = last_id = response = None
4478         for page_num in itertools.count(1):
4479             videos = list(self._playlist_entries(playlist))
4480             if not videos:
4481                 return
4482             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4483             if start >= len(videos):
4484                 return
4485             yield from videos[start:]
4486             first_id = first_id or videos[0]['id']
4487             last_id = videos[-1]['id']
4488             watch_endpoint = try_get(
4489                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4490             headers = self.generate_api_headers(
4491                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4492                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4493             query = {
4494                 'playlistId': playlist_id,
4495                 'videoId': watch_endpoint.get('videoId') or last_id,
4496                 'index': watch_endpoint.get('index') or len(videos),
4497                 'params': watch_endpoint.get('params') or 'OAE%3D'
4498             }
4499             response = self._extract_response(
4500                 item_id='%s page %d' % (playlist_id, page_num),
4501                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4502                 check_get_keys='contents'
4503             )
4504             playlist = try_get(
4505                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4506
4507     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4508         title = playlist.get('title') or try_get(
4509             data, lambda x: x['titleText']['simpleText'], str)
4510         playlist_id = playlist.get('playlistId') or item_id
4511
4512         # Delegating everything except mix playlists to regular tab-based playlist URL
4513         playlist_url = urljoin(url, try_get(
4514             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4515             str))
4516
4517         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4518         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4519         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4520
4521         if playlist_url and playlist_url != url and not is_known_unviewable:
4522             return self.url_result(
4523                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4524                 video_title=title)
4525
4526         return self.playlist_result(
4527             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4528             playlist_id=playlist_id, playlist_title=title)
4529
4530     def _extract_availability(self, data):
4531         """
4532         Gets the availability of a given playlist/tab.
4533         Note: Unless YouTube tells us explicitly, we do not assume it is public
4534         @param data: response
4535         """
4536         is_private = is_unlisted = None
4537         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4538         badge_labels = self._extract_badges(renderer)
4539
4540         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4541         privacy_dropdown_entries = try_get(
4542             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4543         for renderer_dict in privacy_dropdown_entries:
4544             is_selected = try_get(
4545                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4546             if not is_selected:
4547                 continue
4548             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4549             if label:
4550                 badge_labels.add(label.lower())
4551                 break
4552
4553         for badge_label in badge_labels:
4554             if badge_label == 'unlisted':
4555                 is_unlisted = True
4556             elif badge_label == 'private':
4557                 is_private = True
4558             elif badge_label == 'public':
4559                 is_unlisted = is_private = False
4560         return self._availability(is_private, False, False, False, is_unlisted)
4561
4562     @staticmethod
4563     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4564         sidebar_renderer = try_get(
4565             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4566         for item in sidebar_renderer:
4567             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4568             if renderer:
4569                 return renderer
4570
4571     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4572         """
4573         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4574         """
4575         browse_id = params = None
4576         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4577         if not renderer:
4578             return
4579         menu_renderer = try_get(
4580             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4581         for menu_item in menu_renderer:
4582             if not isinstance(menu_item, dict):
4583                 continue
4584             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4585             text = try_get(
4586                 nav_item_renderer, lambda x: x['text']['simpleText'], str)
4587             if not text or text.lower() != 'show unavailable videos':
4588                 continue
4589             browse_endpoint = try_get(
4590                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4591             browse_id = browse_endpoint.get('browseId')
4592             params = browse_endpoint.get('params')
4593             break
4594
4595         headers = self.generate_api_headers(
4596             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4597             visitor_data=self._extract_visitor_data(data, ytcfg))
4598         query = {
4599             'params': params or 'wgYCCAA=',
4600             'browseId': browse_id or 'VL%s' % item_id
4601         }
4602         return self._extract_response(
4603             item_id=item_id, headers=headers, query=query,
4604             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4605             note='Downloading API JSON with unavailable videos')
4606
4607     @functools.cached_property
4608     def skip_webpage(self):
4609         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4610
4611     def _extract_webpage(self, url, item_id, fatal=True):
4612         webpage, data = None, None
4613         for retry in self.RetryManager(fatal=fatal):
4614             try:
4615                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
4616                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4617             except ExtractorError as e:
4618                 if isinstance(e.cause, network_exceptions):
4619                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
4620                         retry.error = e
4621                         continue
4622                 self._error_or_warning(e, fatal=fatal)
4623                 break
4624
4625             try:
4626                 self._extract_and_report_alerts(data)
4627             except ExtractorError as e:
4628                 self._error_or_warning(e, fatal=fatal)
4629                 break
4630
4631             # Sometimes youtube returns a webpage with incomplete ytInitialData
4632             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4633             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
4634                 retry.error = ExtractorError('Incomplete yt initial data received')
4635                 continue
4636
4637         return webpage, data
4638
4639     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4640         """Use if failed to extract ytcfg (and data) from initial webpage"""
4641         if not ytcfg and self.is_authenticated:
4642             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4643             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4644                 raise ExtractorError(
4645                     f'{msg}. If you are not downloading private content, or '
4646                     'your cookies are only for the first account and channel,'
4647                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4648                     expected=True)
4649             self.report_warning(msg, only_once=True)
4650
4651     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4652         data = None
4653         if not self.skip_webpage:
4654             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4655             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4656             # Reject webpage data if redirected to home page without explicitly requesting
4657             selected_tab = self._extract_selected_tab(traverse_obj(
4658                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4659             if (url != 'https://www.youtube.com/feed/recommended'
4660                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4661                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4662                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4663                 if fatal:
4664                     raise ExtractorError(msg, expected=True)
4665                 self.report_warning(msg, only_once=True)
4666         if not data:
4667             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4668             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4669         return data, ytcfg
4670
4671     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4672         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4673         resolve_response = self._extract_response(
4674             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4675             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4676         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4677         for ep_key, ep in endpoints.items():
4678             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4679             if params:
4680                 return self._extract_response(
4681                     item_id=item_id, query=params, ep=ep, headers=headers,
4682                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4683                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4684         err_note = 'Failed to resolve url (does the playlist exist?)'
4685         if fatal:
4686             raise ExtractorError(err_note, expected=True)
4687         self.report_warning(err_note, item_id)
4688
4689     _SEARCH_PARAMS = None
4690
4691     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4692         data = {'query': query}
4693         if params is NO_DEFAULT:
4694             params = self._SEARCH_PARAMS
4695         if params:
4696             data['params'] = params
4697
4698         content_keys = (
4699             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4700             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4701             # ytmusic search
4702             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4703             ('continuationContents', ),
4704         )
4705         display_id = f'query "{query}"'
4706         check_get_keys = tuple({keys[0] for keys in content_keys})
4707         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4708         self._report_playlist_authcheck(ytcfg, fatal=False)
4709
4710         continuation_list = [None]
4711         search = None
4712         for page_num in itertools.count(1):
4713             data.update(continuation_list[0] or {})
4714             headers = self.generate_api_headers(
4715                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4716             search = self._extract_response(
4717                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4718                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4719             slr_contents = traverse_obj(search, *content_keys)
4720             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4721             if not continuation_list[0]:
4722                 break
4723
4724
4725 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4726     IE_DESC = 'YouTube Tabs'
4727     _VALID_URL = r'''(?x:
4728         https?://
4729             (?:\w+\.)?
4730             (?:
4731                 youtube(?:kids)?\.com|
4732                 %(invidious)s
4733             )/
4734             (?:
4735                 (?P<channel_type>channel|c|user|browse)/|
4736                 (?P<not_channel>
4737                     feed/|hashtag/|
4738                     (?:playlist|watch)\?.*?\blist=
4739                 )|
4740                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4741             )
4742             (?P<id>[^/?\#&]+)
4743     )''' % {
4744         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4745         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4746     }
4747     IE_NAME = 'youtube:tab'
4748
4749     _TESTS = [{
4750         'note': 'playlists, multipage',
4751         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4752         'playlist_mincount': 94,
4753         'info_dict': {
4754             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4755             'title': 'Igor Kleiner - Playlists',
4756             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4757             'uploader': 'Igor Kleiner',
4758             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4759             'channel': 'Igor Kleiner',
4760             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4761             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4762             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4763             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4764             'channel_follower_count': int
4765         },
4766     }, {
4767         'note': 'playlists, multipage, different order',
4768         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4769         'playlist_mincount': 94,
4770         'info_dict': {
4771             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4772             'title': 'Igor Kleiner - Playlists',
4773             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4774             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4775             'uploader': 'Igor Kleiner',
4776             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4777             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4778             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4779             'channel': 'Igor Kleiner',
4780             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4781             'channel_follower_count': int
4782         },
4783     }, {
4784         'note': 'playlists, series',
4785         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4786         'playlist_mincount': 5,
4787         'info_dict': {
4788             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4789             'title': '3Blue1Brown - Playlists',
4790             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4791             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4792             'uploader': '3Blue1Brown',
4793             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4794             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4795             'channel': '3Blue1Brown',
4796             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4797             'tags': ['Mathematics'],
4798             'channel_follower_count': int
4799         },
4800     }, {
4801         'note': 'playlists, singlepage',
4802         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4803         'playlist_mincount': 4,
4804         'info_dict': {
4805             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4806             'title': 'ThirstForScience - Playlists',
4807             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4808             'uploader': 'ThirstForScience',
4809             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4810             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4811             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4812             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4813             'tags': 'count:13',
4814             'channel': 'ThirstForScience',
4815             'channel_follower_count': int
4816         }
4817     }, {
4818         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4819         'only_matching': True,
4820     }, {
4821         'note': 'basic, single video playlist',
4822         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4823         'info_dict': {
4824             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4825             'uploader': 'Sergey M.',
4826             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4827             'title': 'youtube-dl public playlist',
4828             'description': '',
4829             'tags': [],
4830             'view_count': int,
4831             'modified_date': '20201130',
4832             'channel': 'Sergey M.',
4833             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4834             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4835             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4836         },
4837         'playlist_count': 1,
4838     }, {
4839         'note': 'empty playlist',
4840         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4841         'info_dict': {
4842             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4843             'uploader': 'Sergey M.',
4844             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4845             'title': 'youtube-dl empty playlist',
4846             'tags': [],
4847             'channel': 'Sergey M.',
4848             'description': '',
4849             'modified_date': '20160902',
4850             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4851             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4852             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4853         },
4854         'playlist_count': 0,
4855     }, {
4856         'note': 'Home tab',
4857         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4858         'info_dict': {
4859             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4860             'title': 'lex will - Home',
4861             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4862             'uploader': 'lex will',
4863             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4864             'channel': 'lex will',
4865             'tags': ['bible', 'history', 'prophesy'],
4866             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4867             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4868             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4869             'channel_follower_count': int
4870         },
4871         'playlist_mincount': 2,
4872     }, {
4873         'note': 'Videos tab',
4874         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4875         'info_dict': {
4876             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4877             'title': 'lex will - Videos',
4878             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4879             'uploader': 'lex will',
4880             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4881             'tags': ['bible', 'history', 'prophesy'],
4882             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4883             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4884             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4885             'channel': 'lex will',
4886             'channel_follower_count': int
4887         },
4888         'playlist_mincount': 975,
4889     }, {
4890         'note': 'Videos tab, sorted by popular',
4891         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4892         'info_dict': {
4893             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4894             'title': 'lex will - Videos',
4895             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4896             'uploader': 'lex will',
4897             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4898             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4899             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4900             'channel': 'lex will',
4901             'tags': ['bible', 'history', 'prophesy'],
4902             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4903             'channel_follower_count': int
4904         },
4905         'playlist_mincount': 199,
4906     }, {
4907         'note': 'Playlists tab',
4908         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4909         'info_dict': {
4910             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4911             'title': 'lex will - Playlists',
4912             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4913             'uploader': 'lex will',
4914             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4915             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4916             'channel': 'lex will',
4917             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4918             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4919             'tags': ['bible', 'history', 'prophesy'],
4920             'channel_follower_count': int
4921         },
4922         'playlist_mincount': 17,
4923     }, {
4924         'note': 'Community tab',
4925         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4926         'info_dict': {
4927             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4928             'title': 'lex will - Community',
4929             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4930             'uploader': 'lex will',
4931             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4932             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4933             'channel': 'lex will',
4934             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4935             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4936             'tags': ['bible', 'history', 'prophesy'],
4937             'channel_follower_count': int
4938         },
4939         'playlist_mincount': 18,
4940     }, {
4941         'note': 'Channels tab',
4942         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4943         'info_dict': {
4944             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4945             'title': 'lex will - Channels',
4946             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4947             'uploader': 'lex will',
4948             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4949             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4950             'channel': 'lex will',
4951             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4952             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4953             'tags': ['bible', 'history', 'prophesy'],
4954             'channel_follower_count': int
4955         },
4956         'playlist_mincount': 12,
4957     }, {
4958         'note': 'Search tab',
4959         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4960         'playlist_mincount': 40,
4961         'info_dict': {
4962             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4963             'title': '3Blue1Brown - Search - linear algebra',
4964             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4965             'uploader': '3Blue1Brown',
4966             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4967             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4968             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4969             'tags': ['Mathematics'],
4970             'channel': '3Blue1Brown',
4971             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4972             'channel_follower_count': int
4973         },
4974     }, {
4975         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4976         'only_matching': True,
4977     }, {
4978         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4979         'only_matching': True,
4980     }, {
4981         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4982         'only_matching': True,
4983     }, {
4984         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4985         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4986         'info_dict': {
4987             'title': '29C3: Not my department',
4988             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4989             'uploader': 'Christiaan008',
4990             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4991             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4992             'tags': [],
4993             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4994             'view_count': int,
4995             'modified_date': '20150605',
4996             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4997             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4998             'channel': 'Christiaan008',
4999         },
5000         'playlist_count': 96,
5001     }, {
5002         'note': 'Large playlist',
5003         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5004         'info_dict': {
5005             'title': 'Uploads from Cauchemar',
5006             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5007             'uploader': 'Cauchemar',
5008             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5009             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5010             'tags': [],
5011             'modified_date': r're:\d{8}',
5012             'channel': 'Cauchemar',
5013             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5014             'view_count': int,
5015             'description': '',
5016             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5017         },
5018         'playlist_mincount': 1123,
5019         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5020     }, {
5021         'note': 'even larger playlist, 8832 videos',
5022         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5023         'only_matching': True,
5024     }, {
5025         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5026         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5027         'info_dict': {
5028             'title': 'Uploads from Interstellar Movie',
5029             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5030             'uploader': 'Interstellar Movie',
5031             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5032             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5033             'tags': [],
5034             'view_count': int,
5035             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5036             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5037             'channel': 'Interstellar Movie',
5038             'description': '',
5039             'modified_date': r're:\d{8}',
5040         },
5041         'playlist_mincount': 21,
5042     }, {
5043         'note': 'Playlist with "show unavailable videos" button',
5044         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5045         'info_dict': {
5046             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5047             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5048             'uploader': 'Phim Siêu Nhân Nhật Bản',
5049             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5050             'view_count': int,
5051             'channel': 'Phim Siêu Nhân Nhật Bản',
5052             'tags': [],
5053             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5054             'description': '',
5055             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5056             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5057             'modified_date': r're:\d{8}',
5058         },
5059         'playlist_mincount': 200,
5060         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5061     }, {
5062         'note': 'Playlist with unavailable videos in page 7',
5063         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5064         'info_dict': {
5065             'title': 'Uploads from BlankTV',
5066             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5067             'uploader': 'BlankTV',
5068             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5069             'channel': 'BlankTV',
5070             'channel_url': 'https://www.youtube.com/c/blanktv',
5071             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5072             'view_count': int,
5073             'tags': [],
5074             'uploader_url': 'https://www.youtube.com/c/blanktv',
5075             'modified_date': r're:\d{8}',
5076             'description': '',
5077         },
5078         'playlist_mincount': 1000,
5079         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5080     }, {
5081         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5082         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5083         'info_dict': {
5084             'title': 'Data Analysis with Dr Mike Pound',
5085             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5086             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5087             'uploader': 'Computerphile',
5088             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5089             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5090             'tags': [],
5091             'view_count': int,
5092             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5093             'channel_url': 'https://www.youtube.com/user/Computerphile',
5094             'channel': 'Computerphile',
5095         },
5096         'playlist_mincount': 11,
5097     }, {
5098         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5099         'only_matching': True,
5100     }, {
5101         'note': 'Playlist URL that does not actually serve a playlist',
5102         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5103         'info_dict': {
5104             'id': 'FqZTN594JQw',
5105             'ext': 'webm',
5106             'title': "Smiley's People 01 detective, Adventure Series, Action",
5107             'uploader': 'STREEM',
5108             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5109             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5110             'upload_date': '20150526',
5111             'license': 'Standard YouTube License',
5112             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5113             'categories': ['People & Blogs'],
5114             'tags': list,
5115             'view_count': int,
5116             'like_count': int,
5117         },
5118         'params': {
5119             'skip_download': True,
5120         },
5121         'skip': 'This video is not available.',
5122         'add_ie': [YoutubeIE.ie_key()],
5123     }, {
5124         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5125         'only_matching': True,
5126     }, {
5127         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5128         'only_matching': True,
5129     }, {
5130         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5131         'info_dict': {
5132             'id': 'Wq15eF5vCbI',  # This will keep changing
5133             'ext': 'mp4',
5134             'title': str,
5135             'uploader': 'Sky News',
5136             'uploader_id': 'skynews',
5137             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5138             'upload_date': r're:\d{8}',
5139             'description': str,
5140             'categories': ['News & Politics'],
5141             'tags': list,
5142             'like_count': int,
5143             'release_timestamp': 1642502819,
5144             'channel': 'Sky News',
5145             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5146             'age_limit': 0,
5147             'view_count': int,
5148             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
5149             'playable_in_embed': True,
5150             'release_date': '20220118',
5151             'availability': 'public',
5152             'live_status': 'is_live',
5153             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5154             'channel_follower_count': int
5155         },
5156         'params': {
5157             'skip_download': True,
5158         },
5159         'expected_warnings': ['Ignoring subtitle tracks found in '],
5160     }, {
5161         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5162         'info_dict': {
5163             'id': 'a48o2S1cPoo',
5164             'ext': 'mp4',
5165             'title': 'The Young Turks - Live Main Show',
5166             'uploader': 'The Young Turks',
5167             'uploader_id': 'TheYoungTurks',
5168             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5169             'upload_date': '20150715',
5170             'license': 'Standard YouTube License',
5171             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5172             'categories': ['News & Politics'],
5173             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5174             'like_count': int,
5175         },
5176         'params': {
5177             'skip_download': True,
5178         },
5179         'only_matching': True,
5180     }, {
5181         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5182         'only_matching': True,
5183     }, {
5184         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5185         'only_matching': True,
5186     }, {
5187         'note': 'A channel that is not live. Should raise error',
5188         'url': 'https://www.youtube.com/user/numberphile/live',
5189         'only_matching': True,
5190     }, {
5191         'url': 'https://www.youtube.com/feed/trending',
5192         'only_matching': True,
5193     }, {
5194         'url': 'https://www.youtube.com/feed/library',
5195         'only_matching': True,
5196     }, {
5197         'url': 'https://www.youtube.com/feed/history',
5198         'only_matching': True,
5199     }, {
5200         'url': 'https://www.youtube.com/feed/subscriptions',
5201         'only_matching': True,
5202     }, {
5203         'url': 'https://www.youtube.com/feed/watch_later',
5204         'only_matching': True,
5205     }, {
5206         'note': 'Recommended - redirects to home page.',
5207         'url': 'https://www.youtube.com/feed/recommended',
5208         'only_matching': True,
5209     }, {
5210         'note': 'inline playlist with not always working continuations',
5211         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5212         'only_matching': True,
5213     }, {
5214         'url': 'https://www.youtube.com/course',
5215         'only_matching': True,
5216     }, {
5217         'url': 'https://www.youtube.com/zsecurity',
5218         'only_matching': True,
5219     }, {
5220         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5221         'only_matching': True,
5222     }, {
5223         'url': 'https://www.youtube.com/TheYoungTurks/live',
5224         'only_matching': True,
5225     }, {
5226         'url': 'https://www.youtube.com/hashtag/cctv9',
5227         'info_dict': {
5228             'id': 'cctv9',
5229             'title': '#cctv9',
5230             'tags': [],
5231         },
5232         'playlist_mincount': 350,
5233     }, {
5234         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5235         'only_matching': True,
5236     }, {
5237         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5238         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5239         'only_matching': True
5240     }, {
5241         'note': '/browse/ should redirect to /channel/',
5242         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5243         'only_matching': True
5244     }, {
5245         'note': 'VLPL, should redirect to playlist?list=PL...',
5246         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5247         'info_dict': {
5248             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5249             'uploader': 'NoCopyrightSounds',
5250             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5251             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5252             'title': 'NCS : All Releases 💿',
5253             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5254             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5255             'modified_date': r're:\d{8}',
5256             'view_count': int,
5257             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5258             'tags': [],
5259             'channel': 'NoCopyrightSounds',
5260         },
5261         'playlist_mincount': 166,
5262         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5263     }, {
5264         'note': 'Topic, should redirect to playlist?list=UU...',
5265         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5266         'info_dict': {
5267             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5268             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5269             'title': 'Uploads from Royalty Free Music - Topic',
5270             'uploader': 'Royalty Free Music - Topic',
5271             'tags': [],
5272             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5273             'channel': 'Royalty Free Music - Topic',
5274             'view_count': int,
5275             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5276             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5277             'modified_date': r're:\d{8}',
5278             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5279             'description': '',
5280         },
5281         'expected_warnings': [
5282             'The URL does not have a videos tab',
5283             r'[Uu]navailable videos (are|will be) hidden',
5284         ],
5285         'playlist_mincount': 101,
5286     }, {
5287         'note': 'Topic without a UU playlist',
5288         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5289         'info_dict': {
5290             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5291             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5292             'tags': [],
5293         },
5294         'expected_warnings': [
5295             'the playlist redirect gave error',
5296         ],
5297         'playlist_mincount': 9,
5298     }, {
5299         'note': 'Youtube music Album',
5300         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5301         'info_dict': {
5302             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5303             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5304             'tags': [],
5305             'view_count': int,
5306             'description': '',
5307             'availability': 'unlisted',
5308             'modified_date': r're:\d{8}',
5309         },
5310         'playlist_count': 50,
5311     }, {
5312         'note': 'unlisted single video playlist',
5313         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5314         'info_dict': {
5315             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5316             'uploader': 'colethedj',
5317             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5318             'title': 'yt-dlp unlisted playlist test',
5319             'availability': 'unlisted',
5320             'tags': [],
5321             'modified_date': '20220418',
5322             'channel': 'colethedj',
5323             'view_count': int,
5324             'description': '',
5325             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5326             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5327             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5328         },
5329         'playlist_count': 1,
5330     }, {
5331         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5332         'url': 'https://www.youtube.com/feed/recommended',
5333         'info_dict': {
5334             'id': 'recommended',
5335             'title': 'recommended',
5336             'tags': [],
5337         },
5338         'playlist_mincount': 50,
5339         'params': {
5340             'skip_download': True,
5341             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5342         },
5343     }, {
5344         'note': 'API Fallback: /videos tab, sorted by oldest first',
5345         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5346         'info_dict': {
5347             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5348             'title': 'Cody\'sLab - Videos',
5349             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5350             'uploader': 'Cody\'sLab',
5351             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5352             'channel': 'Cody\'sLab',
5353             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5354             'tags': [],
5355             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5356             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5357             'channel_follower_count': int
5358         },
5359         'playlist_mincount': 650,
5360         'params': {
5361             'skip_download': True,
5362             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5363         },
5364     }, {
5365         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5366         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5367         'info_dict': {
5368             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5369             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5370             'title': 'Uploads from Royalty Free Music - Topic',
5371             'uploader': 'Royalty Free Music - Topic',
5372             'modified_date': r're:\d{8}',
5373             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5374             'description': '',
5375             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5376             'tags': [],
5377             'channel': 'Royalty Free Music - Topic',
5378             'view_count': int,
5379             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5380         },
5381         'expected_warnings': [
5382             'does not have a videos tab',
5383             r'[Uu]navailable videos (are|will be) hidden',
5384         ],
5385         'playlist_mincount': 101,
5386         'params': {
5387             'skip_download': True,
5388             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5389         },
5390     }, {
5391         'note': 'non-standard redirect to regional channel',
5392         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5393         'only_matching': True
5394     }, {
5395         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5396         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5397         'info_dict': {
5398             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5399             'modified_date': '20220407',
5400             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5401             'tags': [],
5402             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5403             'uploader': 'pukkandan',
5404             'availability': 'unlisted',
5405             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5406             'channel': 'pukkandan',
5407             'description': 'Test for collaborative playlist',
5408             'title': 'yt-dlp test - collaborative playlist',
5409             'view_count': int,
5410             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5411         },
5412         'playlist_mincount': 2
5413     }]
5414
5415     @classmethod
5416     def suitable(cls, url):
5417         return False if YoutubeIE.suitable(url) else super().suitable(url)
5418
5419     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5420
5421     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5422     def _real_extract(self, url, smuggled_data):
5423         item_id = self._match_id(url)
5424         url = urllib.parse.urlunparse(
5425             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
5426         compat_opts = self.get_param('compat_opts', [])
5427
5428         def get_mobj(url):
5429             mobj = self._URL_RE.match(url).groupdict()
5430             mobj.update((k, '') for k, v in mobj.items() if v is None)
5431             return mobj
5432
5433         mobj, redirect_warning = get_mobj(url), None
5434         # Youtube returns incomplete data if tabname is not lower case
5435         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5436         if is_channel:
5437             if smuggled_data.get('is_music_url'):
5438                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5439                     item_id = item_id[2:]
5440                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5441                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5442                     mdata = self._extract_tab_endpoint(
5443                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5444                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5445                                         get_all=False, expected_type=str)
5446                     if not murl:
5447                         raise ExtractorError('Failed to resolve album to playlist')
5448                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5449                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5450                     pre = f'https://www.youtube.com/channel/{item_id}'
5451
5452         original_tab_name = tab
5453         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5454             # Home URLs should redirect to /videos/
5455             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5456                                 'To download only the videos in the home page, add a "/featured" to the URL')
5457             tab = '/videos'
5458
5459         url = ''.join((pre, tab, post))
5460         mobj = get_mobj(url)
5461
5462         # Handle both video/playlist URLs
5463         qs = parse_qs(url)
5464         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5465
5466         if not video_id and mobj['not_channel'].startswith('watch'):
5467             if not playlist_id:
5468                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5469                 raise ExtractorError('Unable to recognize tab page')
5470             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5471             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5472             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5473             mobj = get_mobj(url)
5474
5475         if video_id and playlist_id:
5476             if self.get_param('noplaylist'):
5477                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5478                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5479                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5480             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5481
5482         data, ytcfg = self._extract_data(url, item_id)
5483
5484         # YouTube may provide a non-standard redirect to the regional channel
5485         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5486         redirect_url = traverse_obj(
5487             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5488         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5489             redirect_url = ''.join((
5490                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5491             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5492             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5493
5494         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5495         if tabs:
5496             selected_tab = self._extract_selected_tab(tabs)
5497             selected_tab_name = selected_tab.get('title', '').lower()
5498             if selected_tab_name == 'home':
5499                 selected_tab_name = 'featured'
5500             requested_tab_name = mobj['tab'][1:]
5501             if 'no-youtube-channel-redirect' not in compat_opts:
5502                 if requested_tab_name == 'live':  # Live tab should have redirected to the video
5503                     raise UserNotLive(video_id=mobj['id'])
5504                 if requested_tab_name not in ('', selected_tab_name):
5505                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5506                     if not original_tab_name:
5507                         if item_id[:2] == 'UC':
5508                             # Topic channels don't have /videos. Use the equivalent playlist instead
5509                             pl_id = f'UU{item_id[2:]}'
5510                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5511                             try:
5512                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5513                             except ExtractorError:
5514                                 redirect_warning += ' and the playlist redirect gave error'
5515                             else:
5516                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5517                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5518                         if selected_tab_name and selected_tab_name != requested_tab_name:
5519                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5520                     else:
5521                         raise ExtractorError(redirect_warning, expected=True)
5522
5523         if redirect_warning:
5524             self.to_screen(redirect_warning)
5525         self.write_debug(f'Final URL: {url}')
5526
5527         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5528         if 'no-youtube-unavailable-videos' not in compat_opts:
5529             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5530         self._extract_and_report_alerts(data, only_once=True)
5531         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5532         if tabs:
5533             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5534
5535         playlist = traverse_obj(
5536             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5537         if playlist:
5538             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5539
5540         video_id = traverse_obj(
5541             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5542         if video_id:
5543             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5544                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5545             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5546                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5547
5548         raise ExtractorError('Unable to recognize tab page')
5549
5550
5551 class YoutubePlaylistIE(InfoExtractor):
5552     IE_DESC = 'YouTube playlists'
5553     _VALID_URL = r'''(?x)(?:
5554                         (?:https?://)?
5555                         (?:\w+\.)?
5556                         (?:
5557                             (?:
5558                                 youtube(?:kids)?\.com|
5559                                 %(invidious)s
5560                             )
5561                             /.*?\?.*?\blist=
5562                         )?
5563                         (?P<id>%(playlist_id)s)
5564                      )''' % {
5565         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5566         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5567     }
5568     IE_NAME = 'youtube:playlist'
5569     _TESTS = [{
5570         'note': 'issue #673',
5571         'url': 'PLBB231211A4F62143',
5572         'info_dict': {
5573             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5574             'id': 'PLBB231211A4F62143',
5575             'uploader': 'Wickman',
5576             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5577             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5578             'view_count': int,
5579             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5580             'modified_date': r're:\d{8}',
5581             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5582             'channel': 'Wickman',
5583             'tags': [],
5584             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5585         },
5586         'playlist_mincount': 29,
5587     }, {
5588         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5589         'info_dict': {
5590             'title': 'YDL_safe_search',
5591             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5592         },
5593         'playlist_count': 2,
5594         'skip': 'This playlist is private',
5595     }, {
5596         'note': 'embedded',
5597         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5598         'playlist_count': 4,
5599         'info_dict': {
5600             'title': 'JODA15',
5601             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5602             'uploader': 'milan',
5603             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5604             'description': '',
5605             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5606             'tags': [],
5607             'modified_date': '20140919',
5608             'view_count': int,
5609             'channel': 'milan',
5610             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5611             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5612         },
5613         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5614     }, {
5615         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5616         'playlist_mincount': 455,
5617         'info_dict': {
5618             'title': '2018 Chinese New Singles (11/6 updated)',
5619             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5620             'uploader': 'LBK',
5621             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5622             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5623             'channel': 'LBK',
5624             'view_count': int,
5625             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5626             'tags': [],
5627             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5628             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5629             'modified_date': r're:\d{8}',
5630         },
5631         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5632     }, {
5633         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5634         'only_matching': True,
5635     }, {
5636         # music album playlist
5637         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5638         'only_matching': True,
5639     }]
5640
5641     @classmethod
5642     def suitable(cls, url):
5643         if YoutubeTabIE.suitable(url):
5644             return False
5645         from ..utils import parse_qs
5646         qs = parse_qs(url)
5647         if qs.get('v', [None])[0]:
5648             return False
5649         return super().suitable(url)
5650
5651     def _real_extract(self, url):
5652         playlist_id = self._match_id(url)
5653         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5654         url = update_url_query(
5655             'https://www.youtube.com/playlist',
5656             parse_qs(url) or {'list': playlist_id})
5657         if is_music_url:
5658             url = smuggle_url(url, {'is_music_url': True})
5659         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5660
5661
5662 class YoutubeYtBeIE(InfoExtractor):
5663     IE_DESC = 'youtu.be'
5664     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5665     _TESTS = [{
5666         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5667         'info_dict': {
5668             'id': 'yeWKywCrFtk',
5669             'ext': 'mp4',
5670             'title': 'Small Scale Baler and Braiding Rugs',
5671             'uploader': 'Backus-Page House Museum',
5672             'uploader_id': 'backuspagemuseum',
5673             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5674             'upload_date': '20161008',
5675             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5676             'categories': ['Nonprofits & Activism'],
5677             'tags': list,
5678             'like_count': int,
5679             'age_limit': 0,
5680             'playable_in_embed': True,
5681             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5682             'channel': 'Backus-Page House Museum',
5683             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5684             'live_status': 'not_live',
5685             'view_count': int,
5686             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5687             'availability': 'public',
5688             'duration': 59,
5689             'comment_count': int,
5690             'channel_follower_count': int
5691         },
5692         'params': {
5693             'noplaylist': True,
5694             'skip_download': True,
5695         },
5696     }, {
5697         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5698         'only_matching': True,
5699     }]
5700
5701     def _real_extract(self, url):
5702         mobj = self._match_valid_url(url)
5703         video_id = mobj.group('id')
5704         playlist_id = mobj.group('playlist_id')
5705         return self.url_result(
5706             update_url_query('https://www.youtube.com/watch', {
5707                 'v': video_id,
5708                 'list': playlist_id,
5709                 'feature': 'youtu.be',
5710             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5711
5712
5713 class YoutubeLivestreamEmbedIE(InfoExtractor):
5714     IE_DESC = 'YouTube livestream embeds'
5715     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5716     _TESTS = [{
5717         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5718         'only_matching': True,
5719     }]
5720
5721     def _real_extract(self, url):
5722         channel_id = self._match_id(url)
5723         return self.url_result(
5724             f'https://www.youtube.com/channel/{channel_id}/live',
5725             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5726
5727
5728 class YoutubeYtUserIE(InfoExtractor):
5729     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5730     IE_NAME = 'youtube:user'
5731     _VALID_URL = r'ytuser:(?P<id>.+)'
5732     _TESTS = [{
5733         'url': 'ytuser:phihag',
5734         'only_matching': True,
5735     }]
5736
5737     def _real_extract(self, url):
5738         user_id = self._match_id(url)
5739         return self.url_result(
5740             'https://www.youtube.com/user/%s/videos' % user_id,
5741             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5742
5743
5744 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5745     IE_NAME = 'youtube:favorites'
5746     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5747     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5748     _LOGIN_REQUIRED = True
5749     _TESTS = [{
5750         'url': ':ytfav',
5751         'only_matching': True,
5752     }, {
5753         'url': ':ytfavorites',
5754         'only_matching': True,
5755     }]
5756
5757     def _real_extract(self, url):
5758         return self.url_result(
5759             'https://www.youtube.com/playlist?list=LL',
5760             ie=YoutubeTabIE.ie_key())
5761
5762
5763 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5764     IE_NAME = 'youtube:notif'
5765     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5766     _VALID_URL = r':ytnotif(?:ication)?s?'
5767     _LOGIN_REQUIRED = True
5768     _TESTS = [{
5769         'url': ':ytnotif',
5770         'only_matching': True,
5771     }, {
5772         'url': ':ytnotifications',
5773         'only_matching': True,
5774     }]
5775
5776     def _extract_notification_menu(self, response, continuation_list):
5777         notification_list = traverse_obj(
5778             response,
5779             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5780             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5781             expected_type=list) or []
5782         continuation_list[0] = None
5783         for item in notification_list:
5784             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5785             if entry:
5786                 yield entry
5787             continuation = item.get('continuationItemRenderer')
5788             if continuation:
5789                 continuation_list[0] = continuation
5790
5791     def _extract_notification_renderer(self, notification):
5792         video_id = traverse_obj(
5793             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5794         url = f'https://www.youtube.com/watch?v={video_id}'
5795         channel_id = None
5796         if not video_id:
5797             browse_ep = traverse_obj(
5798                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5799             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5800             post_id = self._search_regex(
5801                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5802                 'post id', default=None)
5803             if not channel_id or not post_id:
5804                 return
5805             # The direct /post url redirects to this in the browser
5806             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5807
5808         channel = traverse_obj(
5809             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5810             expected_type=str)
5811         notification_title = self._get_text(notification, 'shortMessage')
5812         if notification_title:
5813             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
5814         # TODO: handle recommended videos
5815         title = self._search_regex(
5816             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
5817             'video title', default=None)
5818         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5819                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5820                        else None)
5821         return {
5822             '_type': 'url',
5823             'url': url,
5824             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5825             'video_id': video_id,
5826             'title': title,
5827             'channel_id': channel_id,
5828             'channel': channel,
5829             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5830             'upload_date': upload_date,
5831         }
5832
5833     def _notification_menu_entries(self, ytcfg):
5834         continuation_list = [None]
5835         response = None
5836         for page in itertools.count(1):
5837             ctoken = traverse_obj(
5838                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5839             response = self._extract_response(
5840                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5841                 ep='notification/get_notification_menu', check_get_keys='actions',
5842                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5843             yield from self._extract_notification_menu(response, continuation_list)
5844             if not continuation_list[0]:
5845                 break
5846
5847     def _real_extract(self, url):
5848         display_id = 'notifications'
5849         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5850         self._report_playlist_authcheck(ytcfg)
5851         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5852
5853
5854 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5855     IE_DESC = 'YouTube search'
5856     IE_NAME = 'youtube:search'
5857     _SEARCH_KEY = 'ytsearch'
5858     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5859     _TESTS = [{
5860         'url': 'ytsearch5:youtube-dl test video',
5861         'playlist_count': 5,
5862         'info_dict': {
5863             'id': 'youtube-dl test video',
5864             'title': 'youtube-dl test video',
5865         }
5866     }]
5867
5868
5869 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5870     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5871     _SEARCH_KEY = 'ytsearchdate'
5872     IE_DESC = 'YouTube search, newest videos first'
5873     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5874     _TESTS = [{
5875         'url': 'ytsearchdate5:youtube-dl test video',
5876         'playlist_count': 5,
5877         'info_dict': {
5878             'id': 'youtube-dl test video',
5879             'title': 'youtube-dl test video',
5880         }
5881     }]
5882
5883
5884 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5885     IE_DESC = 'YouTube search URLs with sorting and filter support'
5886     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5887     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5888     _TESTS = [{
5889         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5890         'playlist_mincount': 5,
5891         'info_dict': {
5892             'id': 'youtube-dl test video',
5893             'title': 'youtube-dl test video',
5894         }
5895     }, {
5896         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5897         'playlist_mincount': 5,
5898         'info_dict': {
5899             'id': 'python',
5900             'title': 'python',
5901         }
5902     }, {
5903         'url': 'https://www.youtube.com/results?search_query=%23cats',
5904         'playlist_mincount': 1,
5905         'info_dict': {
5906             'id': '#cats',
5907             'title': '#cats',
5908             # The test suite does not have support for nested playlists
5909             # 'entries': [{
5910             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5911             #     'title': '#cats',
5912             # }],
5913         },
5914     }, {
5915         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5916         'only_matching': True,
5917     }]
5918
5919     def _real_extract(self, url):
5920         qs = parse_qs(url)
5921         query = (qs.get('search_query') or qs.get('q'))[0]
5922         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5923
5924
5925 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5926     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
5927     IE_NAME = 'youtube:music:search_url'
5928     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5929     _TESTS = [{
5930         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5931         'playlist_count': 16,
5932         'info_dict': {
5933             'id': 'royalty free music',
5934             'title': 'royalty free music',
5935         }
5936     }, {
5937         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5938         'playlist_mincount': 30,
5939         'info_dict': {
5940             'id': 'royalty free music - songs',
5941             'title': 'royalty free music - songs',
5942         },
5943         'params': {'extract_flat': 'in_playlist'}
5944     }, {
5945         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5946         'playlist_mincount': 30,
5947         'info_dict': {
5948             'id': 'royalty free music - community playlists',
5949             'title': 'royalty free music - community playlists',
5950         },
5951         'params': {'extract_flat': 'in_playlist'}
5952     }]
5953
5954     _SECTIONS = {
5955         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5956         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5957         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5958         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5959         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5960         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5961     }
5962
5963     def _real_extract(self, url):
5964         qs = parse_qs(url)
5965         query = (qs.get('search_query') or qs.get('q'))[0]
5966         params = qs.get('sp', (None,))[0]
5967         if params:
5968             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5969         else:
5970             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
5971             params = self._SECTIONS.get(section)
5972             if not params:
5973                 section = None
5974         title = join_nonempty(query, section, delim=' - ')
5975         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5976
5977
5978 class YoutubeFeedsInfoExtractor(InfoExtractor):
5979     """
5980     Base class for feed extractors
5981     Subclasses must re-define the _FEED_NAME property.
5982     """
5983     _LOGIN_REQUIRED = True
5984     _FEED_NAME = 'feeds'
5985
5986     def _real_initialize(self):
5987         YoutubeBaseInfoExtractor._check_login_required(self)
5988
5989     @classproperty
5990     def IE_NAME(self):
5991         return f'youtube:{self._FEED_NAME}'
5992
5993     def _real_extract(self, url):
5994         return self.url_result(
5995             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5996
5997
5998 class YoutubeWatchLaterIE(InfoExtractor):
5999     IE_NAME = 'youtube:watchlater'
6000     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6001     _VALID_URL = r':ytwatchlater'
6002     _TESTS = [{
6003         'url': ':ytwatchlater',
6004         'only_matching': True,
6005     }]
6006
6007     def _real_extract(self, url):
6008         return self.url_result(
6009             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6010
6011
6012 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6013     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6014     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6015     _FEED_NAME = 'recommended'
6016     _LOGIN_REQUIRED = False
6017     _TESTS = [{
6018         'url': ':ytrec',
6019         'only_matching': True,
6020     }, {
6021         'url': ':ytrecommended',
6022         'only_matching': True,
6023     }, {
6024         'url': 'https://youtube.com',
6025         'only_matching': True,
6026     }]
6027
6028
6029 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6030     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6031     _VALID_URL = r':ytsub(?:scription)?s?'
6032     _FEED_NAME = 'subscriptions'
6033     _TESTS = [{
6034         'url': ':ytsubs',
6035         'only_matching': True,
6036     }, {
6037         'url': ':ytsubscriptions',
6038         'only_matching': True,
6039     }]
6040
6041
6042 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6043     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6044     _VALID_URL = r':ythis(?:tory)?'
6045     _FEED_NAME = 'history'
6046     _TESTS = [{
6047         'url': ':ythistory',
6048         'only_matching': True,
6049     }]
6050
6051
6052 class YoutubeStoriesIE(InfoExtractor):
6053     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6054     IE_NAME = 'youtube:stories'
6055     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6056     _TESTS = [{
6057         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6058         'only_matching': True,
6059     }]
6060
6061     def _real_extract(self, url):
6062         playlist_id = f'RLTD{self._match_id(url)}'
6063         return self.url_result(
6064             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6065             ie=YoutubeTabIE, video_id=playlist_id)
6066
6067
6068 class YoutubeTruncatedURLIE(InfoExtractor):
6069     IE_NAME = 'youtube:truncated_url'
6070     IE_DESC = False  # Do not list
6071     _VALID_URL = r'''(?x)
6072         (?:https?://)?
6073         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6074         (?:watch\?(?:
6075             feature=[a-z_]+|
6076             annotation_id=annotation_[^&]+|
6077             x-yt-cl=[0-9]+|
6078             hl=[^&]*|
6079             t=[0-9]+
6080         )?
6081         |
6082             attribution_link\?a=[^&]+
6083         )
6084         $
6085     '''
6086
6087     _TESTS = [{
6088         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6089         'only_matching': True,
6090     }, {
6091         'url': 'https://www.youtube.com/watch?',
6092         'only_matching': True,
6093     }, {
6094         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6095         'only_matching': True,
6096     }, {
6097         'url': 'https://www.youtube.com/watch?feature=foo',
6098         'only_matching': True,
6099     }, {
6100         'url': 'https://www.youtube.com/watch?hl=en-GB',
6101         'only_matching': True,
6102     }, {
6103         'url': 'https://www.youtube.com/watch?t=2372',
6104         'only_matching': True,
6105     }]
6106
6107     def _real_extract(self, url):
6108         raise ExtractorError(
6109             'Did you forget to quote the URL? Remember that & is a meta '
6110             'character in most shells, so you want to put the URL in quotes, '
6111             'like  youtube-dl '
6112             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6113             ' or simply  youtube-dl BaW_jenozKc  .',
6114             expected=True)
6115
6116
6117 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6118     IE_NAME = 'youtube:clip'
6119     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6120     _TESTS = [{
6121         # FIXME: Other metadata should be extracted from the clip, not from the base video
6122         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6123         'info_dict': {
6124             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6125             'ext': 'mp4',
6126             'section_start': 29.0,
6127             'section_end': 39.7,
6128             'duration': 10.7,
6129             'age_limit': 0,
6130             'availability': 'public',
6131             'categories': ['Gaming'],
6132             'channel': 'Scott The Woz',
6133             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6134             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6135             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6136             'like_count': int,
6137             'playable_in_embed': True,
6138             'tags': 'count:17',
6139             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6140             'title': 'Mobile Games on Console - Scott The Woz',
6141             'upload_date': '20210920',
6142             'uploader': 'Scott The Woz',
6143             'uploader_id': 'scottthewoz',
6144             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6145             'view_count': int,
6146             'live_status': 'not_live',
6147             'channel_follower_count': int
6148         }
6149     }]
6150
6151     def _real_extract(self, url):
6152         clip_id = self._match_id(url)
6153         _, data = self._extract_webpage(url, clip_id)
6154
6155         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6156         if not video_id:
6157             raise ExtractorError('Unable to find video ID')
6158
6159         clip_data = traverse_obj(data, (
6160             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6161             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6162             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6163             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6164
6165         return {
6166             '_type': 'url_transparent',
6167             'url': f'https://www.youtube.com/watch?v={video_id}',
6168             'ie_key': YoutubeIE.ie_key(),
6169             'id': clip_id,
6170             'section_start': int(clip_data['startTimeMs']) / 1000,
6171             'section_end': int(clip_data['endTimeMs']) / 1000,
6172         }
6173
6174
6175 class YoutubeTruncatedIDIE(InfoExtractor):
6176     IE_NAME = 'youtube:truncated_id'
6177     IE_DESC = False  # Do not list
6178     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
6179
6180     _TESTS = [{
6181         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
6182         'only_matching': True,
6183     }]
6184
6185     def _real_extract(self, url):
6186         video_id = self._match_id(url)
6187         raise ExtractorError(
6188             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
6189             expected=True)