yt_dlp/extractor/youtube.py

   1 import calendar
   2 import copy
   3 import datetime
   4 import functools
   5 import hashlib
   6 import itertools
   7 import json
   8 import math
   9 import os.path
  10 import random
  11 import re
  12 import sys
  13 import time
  14 import traceback
  15 import threading
  16
  17 from .common import InfoExtractor, SearchInfoExtractor
  18 from ..compat import (
  19     compat_chr,
  20     compat_HTTPError,
  21     compat_parse_qs,
  22     compat_str,
  23     compat_urllib_parse_unquote_plus,
  24     compat_urllib_parse_urlencode,
  25     compat_urllib_parse_urlparse,
  26     compat_urlparse,
  27 )
  28 from ..jsinterp import JSInterpreter
  29 from ..utils import (
  30     bug_reports_message,
  31     clean_html,
  32     datetime_from_str,
  33     dict_get,
  34     error_to_compat_str,
  35     ExtractorError,
  36     float_or_none,
  37     format_field,
  38     get_first,
  39     int_or_none,
  40     is_html,
  41     join_nonempty,
  42     js_to_json,
  43     mimetype2ext,
  44     network_exceptions,
  45     NO_DEFAULT,
  46     orderedSet,
  47     parse_codecs,
  48     parse_count,
  49     parse_duration,
  50     parse_iso8601,
  51     parse_qs,
  52     qualities,
  53     remove_end,
  54     remove_start,
  55     smuggle_url,
  56     str_or_none,
  57     str_to_int,
  58     strftime_or_none,
  59     traverse_obj,
  60     try_get,
  61     unescapeHTML,
  62     unified_strdate,
  63     unified_timestamp,
  64     unsmuggle_url,
  65     update_url_query,
  66     url_or_none,
  67     urljoin,
  68     variadic,
  69 )
  70
  71
  72 # any clients starting with _ cannot be explicity requested by the user
  73 INNERTUBE_CLIENTS = {
  74     'web': {
  75         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  76         'INNERTUBE_CONTEXT': {
  77             'client': {
  78                 'clientName': 'WEB',
  79                 'clientVersion': '2.20211221.00.00',
  80             }
  81         },
  82         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  83     },
  84     'web_embedded': {
  85         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  86         'INNERTUBE_CONTEXT': {
  87             'client': {
  88                 'clientName': 'WEB_EMBEDDED_PLAYER',
  89                 'clientVersion': '1.20211215.00.01',
  90             },
  91         },
  92         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  93     },
  94     'web_music': {
  95         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  96         'INNERTUBE_HOST': 'music.youtube.com',
  97         'INNERTUBE_CONTEXT': {
  98             'client': {
  99                 'clientName': 'WEB_REMIX',
 100                 'clientVersion': '1.20211213.00.00',
 101             }
 102         },
 103         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 104     },
 105     'web_creator': {
 106         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 107         'INNERTUBE_CONTEXT': {
 108             'client': {
 109                 'clientName': 'WEB_CREATOR',
 110                 'clientVersion': '1.20211220.02.00',
 111             }
 112         },
 113         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 114     },
 115     'android': {
 116         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 117         'INNERTUBE_CONTEXT': {
 118             'client': {
 119                 'clientName': 'ANDROID',
 120                 'clientVersion': '16.49',
 121             }
 122         },
 123         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 124         'REQUIRE_JS_PLAYER': False
 125     },
 126     'android_embedded': {
 127         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 128         'INNERTUBE_CONTEXT': {
 129             'client': {
 130                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 131                 'clientVersion': '16.49',
 132             },
 133         },
 134         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 135         'REQUIRE_JS_PLAYER': False
 136     },
 137     'android_music': {
 138         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 139         'INNERTUBE_CONTEXT': {
 140             'client': {
 141                 'clientName': 'ANDROID_MUSIC',
 142                 'clientVersion': '4.57',
 143             }
 144         },
 145         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 146         'REQUIRE_JS_PLAYER': False
 147     },
 148     'android_creator': {
 149         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 150         'INNERTUBE_CONTEXT': {
 151             'client': {
 152                 'clientName': 'ANDROID_CREATOR',
 153                 'clientVersion': '21.47',
 154             },
 155         },
 156         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 157         'REQUIRE_JS_PLAYER': False
 158     },
 159     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 160     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 161     'ios': {
 162         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 163         'INNERTUBE_CONTEXT': {
 164             'client': {
 165                 'clientName': 'IOS',
 166                 'clientVersion': '16.46',
 167                 'deviceModel': 'iPhone14,3',
 168             }
 169         },
 170         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 171         'REQUIRE_JS_PLAYER': False
 172     },
 173     'ios_embedded': {
 174         'INNERTUBE_CONTEXT': {
 175             'client': {
 176                 'clientName': 'IOS_MESSAGES_EXTENSION',
 177                 'clientVersion': '16.46',
 178                 'deviceModel': 'iPhone14,3',
 179             },
 180         },
 181         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 182         'REQUIRE_JS_PLAYER': False
 183     },
 184     'ios_music': {
 185         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 186         'INNERTUBE_CONTEXT': {
 187             'client': {
 188                 'clientName': 'IOS_MUSIC',
 189                 'clientVersion': '4.57',
 190             },
 191         },
 192         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 193         'REQUIRE_JS_PLAYER': False
 194     },
 195     'ios_creator': {
 196         'INNERTUBE_CONTEXT': {
 197             'client': {
 198                 'clientName': 'IOS_CREATOR',
 199                 'clientVersion': '21.47',
 200             },
 201         },
 202         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 203         'REQUIRE_JS_PLAYER': False
 204     },
 205     # mweb has 'ultralow' formats
 206     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 207     'mweb': {
 208         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 209         'INNERTUBE_CONTEXT': {
 210             'client': {
 211                 'clientName': 'MWEB',
 212                 'clientVersion': '2.20211221.01.00',
 213             }
 214         },
 215         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 216     },
 217     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 218     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 219     'tv_embedded': {
 220         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 221         'INNERTUBE_CONTEXT': {
 222             'client': {
 223                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 224                 'clientVersion': '2.0',
 225             },
 226         },
 227         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 228     },
 229 }
 230
 231
 232 def _split_innertube_client(client_name):
 233     variant, *base = client_name.rsplit('.', 1)
 234     if base:
 235         return variant, base[0], variant
 236     base, *variant = client_name.split('_', 1)
 237     return client_name, base, variant[0] if variant else None
 238
 239
 240 def build_innertube_clients():
 241     THIRD_PARTY = {
 242         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 243     }
 244     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 245     priority = qualities(BASE_CLIENTS[::-1])
 246
 247     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 248         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 249         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 250         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 251         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 252
 253         _, base_client, variant = _split_innertube_client(client)
 254         ytcfg['priority'] = 10 * priority(base_client)
 255
 256         if not variant:
 257             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 258             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 259             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 260             embedscreen['priority'] -= 3
 261         elif variant == 'embedded':
 262             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 263             ytcfg['priority'] -= 2
 264         else:
 265             ytcfg['priority'] -= 3
 266
 267
 268 build_innertube_clients()
 269
 270
 271 class YoutubeBaseInfoExtractor(InfoExtractor):
 272     """Provide base functions for Youtube extractors"""
 273
 274     _RESERVED_NAMES = (
 275         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 276         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 277         r'browse|oembed|get_video_info|iframe_api|s/player|'
 278         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 279
 280     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 281
 282     # _NETRC_MACHINE = 'youtube'
 283
 284     # If True it will raise an error if no login info is provided
 285     _LOGIN_REQUIRED = False
 286
 287     _INVIDIOUS_SITES = (
 288         # invidious-redirect websites
 289         r'(?:www\.)?redirect\.invidious\.io',
 290         r'(?:(?:www|dev)\.)?invidio\.us',
 291         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
 292         r'(?:www\.)?invidious\.pussthecat\.org',
 293         r'(?:www\.)?invidious\.zee\.li',
 294         r'(?:www\.)?invidious\.ethibox\.fr',
 295         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 296         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 297         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 298         # youtube-dl invidious instances list
 299         r'(?:(?:www|no)\.)?invidiou\.sh',
 300         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 301         r'(?:www\.)?invidious\.kabi\.tk',
 302         r'(?:www\.)?invidious\.mastodon\.host',
 303         r'(?:www\.)?invidious\.zapashcanon\.fr',
 304         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 305         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 306         r'(?:www\.)?invidious\.himiko\.cloud',
 307         r'(?:www\.)?invidious\.reallyancient\.tech',
 308         r'(?:www\.)?invidious\.tube',
 309         r'(?:www\.)?invidiou\.site',
 310         r'(?:www\.)?invidious\.site',
 311         r'(?:www\.)?invidious\.xyz',
 312         r'(?:www\.)?invidious\.nixnet\.xyz',
 313         r'(?:www\.)?invidious\.048596\.xyz',
 314         r'(?:www\.)?invidious\.drycat\.fr',
 315         r'(?:www\.)?inv\.skyn3t\.in',
 316         r'(?:www\.)?tube\.poal\.co',
 317         r'(?:www\.)?tube\.connect\.cafe',
 318         r'(?:www\.)?vid\.wxzm\.sx',
 319         r'(?:www\.)?vid\.mint\.lgbt',
 320         r'(?:www\.)?vid\.puffyan\.us',
 321         r'(?:www\.)?yewtu\.be',
 322         r'(?:www\.)?yt\.elukerio\.org',
 323         r'(?:www\.)?yt\.lelux\.fi',
 324         r'(?:www\.)?invidious\.ggc-project\.de',
 325         r'(?:www\.)?yt\.maisputain\.ovh',
 326         r'(?:www\.)?ytprivate\.com',
 327         r'(?:www\.)?invidious\.13ad\.de',
 328         r'(?:www\.)?invidious\.toot\.koeln',
 329         r'(?:www\.)?invidious\.fdn\.fr',
 330         r'(?:www\.)?watch\.nettohikari\.com',
 331         r'(?:www\.)?invidious\.namazso\.eu',
 332         r'(?:www\.)?invidious\.silkky\.cloud',
 333         r'(?:www\.)?invidious\.exonip\.de',
 334         r'(?:www\.)?invidious\.riverside\.rocks',
 335         r'(?:www\.)?invidious\.blamefran\.net',
 336         r'(?:www\.)?invidious\.moomoo\.de',
 337         r'(?:www\.)?ytb\.trom\.tf',
 338         r'(?:www\.)?yt\.cyberhost\.uk',
 339         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 340         r'(?:www\.)?qklhadlycap4cnod\.onion',
 341         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 342         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 343         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 344         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 345         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 346         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 347         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 348         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 349         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 350         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 351     )
 352
 353     def _initialize_consent(self):
 354         cookies = self._get_cookies('https://www.youtube.com/')
 355         if cookies.get('__Secure-3PSID'):
 356             return
 357         consent_id = None
 358         consent = cookies.get('CONSENT')
 359         if consent:
 360             if 'YES' in consent.value:
 361                 return
 362             consent_id = self._search_regex(
 363                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 364         if not consent_id:
 365             consent_id = random.randint(100, 999)
 366         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 367
 368     def _initialize_pref(self):
 369         cookies = self._get_cookies('https://www.youtube.com/')
 370         pref_cookie = cookies.get('PREF')
 371         pref = {}
 372         if pref_cookie:
 373             try:
 374                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 375             except ValueError:
 376                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 377         pref.update({'hl': 'en', 'tz': 'UTC'})
 378         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 379
 380     def _real_initialize(self):
 381         self._initialize_pref()
 382         self._initialize_consent()
 383         self._check_login_required()
 384
 385     def _check_login_required(self):
 386         if (self._LOGIN_REQUIRED
 387                 and self.get_param('cookiefile') is None
 388                 and self.get_param('cookiesfrombrowser') is None):
 389             self.raise_login_required('Login details are needed to download this content', method='cookies')
 390
 391     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 392     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 393     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 394
 395     def _get_default_ytcfg(self, client='web'):
 396         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 397
 398     def _get_innertube_host(self, client='web'):
 399         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 400
 401     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 402         # try_get but with fallback to default ytcfg client values when present
 403         _func = lambda y: try_get(y, getter, expected_type)
 404         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 405
 406     def _extract_client_name(self, ytcfg, default_client='web'):
 407         return self._ytcfg_get_safe(
 408             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 409                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 410
 411     def _extract_client_version(self, ytcfg, default_client='web'):
 412         return self._ytcfg_get_safe(
 413             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 414                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 415
 416     def _extract_api_key(self, ytcfg=None, default_client='web'):
 417         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 418
 419     def _extract_context(self, ytcfg=None, default_client='web'):
 420         context = get_first(
 421             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 422         # Enforce language and tz for extraction
 423         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 424         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 425         return context
 426
 427     _SAPISID = None
 428
 429     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 430         time_now = round(time.time())
 431         if self._SAPISID is None:
 432             yt_cookies = self._get_cookies('https://www.youtube.com')
 433             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 434             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 435             sapisid_cookie = dict_get(
 436                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 437             if sapisid_cookie and sapisid_cookie.value:
 438                 self._SAPISID = sapisid_cookie.value
 439                 self.write_debug('Extracted SAPISID cookie')
 440                 # SAPISID cookie is required if not already present
 441                 if not yt_cookies.get('SAPISID'):
 442                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 443                     self._set_cookie(
 444                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 445             else:
 446                 self._SAPISID = False
 447         if not self._SAPISID:
 448             return None
 449         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 450         sapisidhash = hashlib.sha1(
 451             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 452         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 453
 454     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 455                   note='Downloading API JSON', errnote='Unable to download API page',
 456                   context=None, api_key=None, api_hostname=None, default_client='web'):
 457
 458         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 459         data.update(query)
 460         real_headers = self.generate_api_headers(default_client=default_client)
 461         real_headers.update({'content-type': 'application/json'})
 462         if headers:
 463             real_headers.update(headers)
 464         return self._download_json(
 465             f'https://{api_hostname or self._get_innertube_host(default_client)}/youtubei/v1/{ep}',
 466             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 467             data=json.dumps(data).encode('utf8'), headers=real_headers,
 468             query={'key': api_key or self._extract_api_key(), 'prettyPrint': 'false'})
 469
 470     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 471         data = self._search_regex(
 472             (fr'{self._YT_INITIAL_DATA_RE}\s*{self._YT_INITIAL_BOUNDARY_RE}',
 473              self._YT_INITIAL_DATA_RE), webpage, 'yt initial data', fatal=fatal)
 474         if data:
 475             return self._parse_json(data, item_id, fatal=fatal)
 476
 477     @staticmethod
 478     def _extract_session_index(*data):
 479         """
 480         Index of current account in account list.
 481         See: https://github.com/yt-dlp/yt-dlp/pull/519
 482         """
 483         for ytcfg in data:
 484             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 485             if session_index is not None:
 486                 return session_index
 487
 488     # Deprecated?
 489     def _extract_identity_token(self, ytcfg=None, webpage=None):
 490         if ytcfg:
 491             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 492             if token:
 493                 return token
 494         if webpage:
 495             return self._search_regex(
 496                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 497                 'identity token', default=None, fatal=False)
 498
 499     @staticmethod
 500     def _extract_account_syncid(*args):
 501         """
 502         Extract syncId required to download private playlists of secondary channels
 503         @params response and/or ytcfg
 504         """
 505         for data in args:
 506             # ytcfg includes channel_syncid if on secondary channel
 507             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 508             if delegated_sid:
 509                 return delegated_sid
 510             sync_ids = (try_get(
 511                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 512                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 513             if len(sync_ids) >= 2 and sync_ids[1]:
 514                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 515                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 516                 return sync_ids[0]
 517
 518     @staticmethod
 519     def _extract_visitor_data(*args):
 520         """
 521         Extracts visitorData from an API response or ytcfg
 522         Appears to be used to track session state
 523         """
 524         return get_first(
 525             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 526             expected_type=str)
 527
 528     @property
 529     def is_authenticated(self):
 530         return bool(self._generate_sapisidhash_header())
 531
 532     def extract_ytcfg(self, video_id, webpage):
 533         if not webpage:
 534             return {}
 535         return self._parse_json(
 536             self._search_regex(
 537                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 538                 default='{}'), video_id, fatal=False) or {}
 539
 540     def generate_api_headers(
 541             self, *, ytcfg=None, account_syncid=None, session_index=None,
 542             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 543
 544         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
 545         headers = {
 546             'X-YouTube-Client-Name': compat_str(
 547                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 548             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 549             'Origin': origin,
 550             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 551             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 552             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 553         }
 554         if session_index is None:
 555             session_index = self._extract_session_index(ytcfg)
 556         if account_syncid or session_index is not None:
 557             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 558
 559         auth = self._generate_sapisidhash_header(origin)
 560         if auth is not None:
 561             headers['Authorization'] = auth
 562             headers['X-Origin'] = origin
 563         return {h: v for h, v in headers.items() if v is not None}
 564
 565     def _download_ytcfg(self, client, video_id):
 566         url = {
 567             'web': 'https://www.youtube.com',
 568             'web_music': 'https://music.youtube.com',
 569             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 570         }.get(client)
 571         if not url:
 572             return {}
 573         webpage = self._download_webpage(
 574             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 575         return self.extract_ytcfg(video_id, webpage) or {}
 576
 577     @staticmethod
 578     def _build_api_continuation_query(continuation, ctp=None):
 579         query = {
 580             'continuation': continuation
 581         }
 582         # TODO: Inconsistency with clickTrackingParams.
 583         # Currently we have a fixed ctp contained within context (from ytcfg)
 584         # and a ctp in root query for continuation.
 585         if ctp:
 586             query['clickTracking'] = {'clickTrackingParams': ctp}
 587         return query
 588
 589     @classmethod
 590     def _extract_next_continuation_data(cls, renderer):
 591         next_continuation = try_get(
 592             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 593                        lambda x: x['continuation']['reloadContinuationData']), dict)
 594         if not next_continuation:
 595             return
 596         continuation = next_continuation.get('continuation')
 597         if not continuation:
 598             return
 599         ctp = next_continuation.get('clickTrackingParams')
 600         return cls._build_api_continuation_query(continuation, ctp)
 601
 602     @classmethod
 603     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 604         if isinstance(continuation_ep, dict):
 605             continuation = try_get(
 606                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 607             if not continuation:
 608                 return
 609             ctp = continuation_ep.get('clickTrackingParams')
 610             return cls._build_api_continuation_query(continuation, ctp)
 611
 612     @classmethod
 613     def _extract_continuation(cls, renderer):
 614         next_continuation = cls._extract_next_continuation_data(renderer)
 615         if next_continuation:
 616             return next_continuation
 617
 618         contents = []
 619         for key in ('contents', 'items'):
 620             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 621
 622         for content in contents:
 623             if not isinstance(content, dict):
 624                 continue
 625             continuation_ep = try_get(
 626                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 627                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 628                 dict)
 629             continuation = cls._extract_continuation_ep_data(continuation_ep)
 630             if continuation:
 631                 return continuation
 632
 633     @classmethod
 634     def _extract_alerts(cls, data):
 635         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 636             if not isinstance(alert_dict, dict):
 637                 continue
 638             for alert in alert_dict.values():
 639                 alert_type = alert.get('type')
 640                 if not alert_type:
 641                     continue
 642                 message = cls._get_text(alert, 'text')
 643                 if message:
 644                     yield alert_type, message
 645
 646     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 647         errors = []
 648         warnings = []
 649         for alert_type, alert_message in alerts:
 650             if alert_type.lower() == 'error' and fatal:
 651                 errors.append([alert_type, alert_message])
 652             else:
 653                 warnings.append([alert_type, alert_message])
 654
 655         for alert_type, alert_message in (warnings + errors[:-1]):
 656             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 657         if errors:
 658             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 659
 660     def _extract_and_report_alerts(self, data, *args, **kwargs):
 661         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 662
 663     def _extract_badges(self, renderer: dict):
 664         badges = set()
 665         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 666             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 667             if label:
 668                 badges.add(label.lower())
 669         return badges
 670
 671     @staticmethod
 672     def _get_text(data, *path_list, max_runs=None):
 673         for path in path_list or [None]:
 674             if path is None:
 675                 obj = [data]
 676             else:
 677                 obj = traverse_obj(data, path, default=[])
 678                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 679                     obj = [obj]
 680             for item in obj:
 681                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 682                 if text:
 683                     return text
 684                 runs = try_get(item, lambda x: x['runs'], list) or []
 685                 if not runs and isinstance(item, list):
 686                     runs = item
 687
 688                 runs = runs[:min(len(runs), max_runs or len(runs))]
 689                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 690                 if text:
 691                     return text
 692
 693     def _get_count(self, data, *path_list):
 694         count_text = self._get_text(data, *path_list) or ''
 695         count = parse_count(count_text)
 696         if count is None:
 697             count = str_to_int(
 698                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 699         return count
 700
 701     @staticmethod
 702     def _extract_thumbnails(data, *path_list):
 703         """
 704         Extract thumbnails from thumbnails dict
 705         @param path_list: path list to level that contains 'thumbnails' key
 706         """
 707         thumbnails = []
 708         for path in path_list or [()]:
 709             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 710                 thumbnail_url = url_or_none(thumbnail.get('url'))
 711                 if not thumbnail_url:
 712                     continue
 713                 # Sometimes youtube gives a wrong thumbnail URL. See:
 714                 # https://github.com/yt-dlp/yt-dlp/issues/233
 715                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 716                 if 'maxresdefault' in thumbnail_url:
 717                     thumbnail_url = thumbnail_url.split('?')[0]
 718                 thumbnails.append({
 719                     'url': thumbnail_url,
 720                     'height': int_or_none(thumbnail.get('height')),
 721                     'width': int_or_none(thumbnail.get('width')),
 722                 })
 723         return thumbnails
 724
 725     @staticmethod
 726     def extract_relative_time(relative_time_text):
 727         """
 728         Extracts a relative time from string and converts to dt object
 729         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 730         """
 731         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 732         if mobj:
 733             start = mobj.group('start')
 734             if start:
 735                 return datetime_from_str(start)
 736             try:
 737                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 738             except ValueError:
 739                 return None
 740
 741     def _extract_time_text(self, renderer, *path_list):
 742         """@returns (timestamp, time_text)"""
 743         text = self._get_text(renderer, *path_list) or ''
 744         dt = self.extract_relative_time(text)
 745         timestamp = None
 746         if isinstance(dt, datetime.datetime):
 747             timestamp = calendar.timegm(dt.timetuple())
 748
 749         if timestamp is None:
 750             timestamp = (
 751                 unified_timestamp(text) or unified_timestamp(
 752                     self._search_regex(
 753                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 754                         text.lower(), 'time text', default=None)))
 755
 756         if text and timestamp is None:
 757             self.report_warning(f"Cannot parse localized time text '{text}'" + bug_reports_message(), only_once=True)
 758         return timestamp, text
 759
 760     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 761                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 762                           default_client='web'):
 763         response = None
 764         last_error = None
 765         count = -1
 766         retries = self.get_param('extractor_retries', 3)
 767         if check_get_keys is None:
 768             check_get_keys = []
 769         while count < retries:
 770             count += 1
 771             if last_error:
 772                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 773             try:
 774                 response = self._call_api(
 775                     ep=ep, fatal=True, headers=headers,
 776                     video_id=item_id, query=query,
 777                     context=self._extract_context(ytcfg, default_client),
 778                     api_key=self._extract_api_key(ytcfg, default_client),
 779                     api_hostname=api_hostname, default_client=default_client,
 780                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 781             except ExtractorError as e:
 782                 if isinstance(e.cause, network_exceptions):
 783                     if isinstance(e.cause, compat_HTTPError):
 784                         first_bytes = e.cause.read(512)
 785                         if not is_html(first_bytes):
 786                             yt_error = try_get(
 787                                 self._parse_json(
 788                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 789                                 lambda x: x['error']['message'], compat_str)
 790                             if yt_error:
 791                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 792                     # Downloading page may result in intermittent 5xx HTTP error
 793                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 794                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 795                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 796                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 797                         last_error = error_to_compat_str(e.cause or e.msg)
 798                         if count < retries:
 799                             continue
 800                 if fatal:
 801                     raise
 802                 else:
 803                     self.report_warning(error_to_compat_str(e))
 804                     return
 805
 806             else:
 807                 try:
 808                     self._extract_and_report_alerts(response, only_once=True)
 809                 except ExtractorError as e:
 810                     # YouTube servers may return errors we want to retry on in a 200 OK response
 811                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 812                     if 'unknown error' in e.msg.lower():
 813                         last_error = e.msg
 814                         continue
 815                     if fatal:
 816                         raise
 817                     self.report_warning(error_to_compat_str(e))
 818                     return
 819                 if not check_get_keys or dict_get(response, check_get_keys):
 820                     break
 821                 # Youtube sometimes sends incomplete data
 822                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 823                 last_error = 'Incomplete data received'
 824                 if count >= retries:
 825                     if fatal:
 826                         raise ExtractorError(last_error)
 827                     else:
 828                         self.report_warning(last_error)
 829                         return
 830         return response
 831
 832     @staticmethod
 833     def is_music_url(url):
 834         return re.match(r'https?://music\.youtube\.com/', url) is not None
 835
 836     def _extract_video(self, renderer):
 837         video_id = renderer.get('videoId')
 838         title = self._get_text(renderer, 'title')
 839         description = self._get_text(renderer, 'descriptionSnippet')
 840         duration = parse_duration(self._get_text(
 841             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 842         if duration is None:
 843             duration = parse_duration(self._search_regex(
 844                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 845                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 846                 video_id, default=None, group='duration'))
 847
 848         view_count = self._get_count(renderer, 'viewCountText')
 849
 850         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 851         channel_id = traverse_obj(
 852             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 853             expected_type=str, get_all=False)
 854         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 855         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 856         overlay_style = traverse_obj(
 857             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 858             get_all=False, expected_type=str)
 859         badges = self._extract_badges(renderer)
 860         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 861         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 862             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 863             expected_type=str)) or ''
 864         url = f'https://www.youtube.com/watch?v={video_id}'
 865         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 866             url = f'https://www.youtube.com/shorts/{video_id}'
 867
 868         return {
 869             '_type': 'url',
 870             'ie_key': YoutubeIE.ie_key(),
 871             'id': video_id,
 872             'url': url,
 873             'title': title,
 874             'description': description,
 875             'duration': duration,
 876             'view_count': view_count,
 877             'uploader': uploader,
 878             'channel_id': channel_id,
 879             'thumbnails': thumbnails,
 880             'upload_date': (strftime_or_none(timestamp, '%Y%m%d')
 881                             if self._configuration_arg('approximate_date', ie_key='youtubetab')
 882                             else None),
 883             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 884                             else 'was_live' if 'streamed' in time_text.lower()
 885                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 886                             else None),
 887             'release_timestamp': scheduled_timestamp,
 888             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 889         }
 890
 891
 892 class YoutubeIE(YoutubeBaseInfoExtractor):
 893     IE_DESC = 'YouTube'
 894     _VALID_URL = r"""(?x)^
 895                      (
 896                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 897                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 898                             (?:www\.)?deturl\.com/www\.youtube\.com|
 899                             (?:www\.)?pwnyoutube\.com|
 900                             (?:www\.)?hooktube\.com|
 901                             (?:www\.)?yourepeat\.com|
 902                             tube\.majestyc\.net|
 903                             %(invidious)s|
 904                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 905                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 906                          (?:                                                  # the various things that can precede the ID:
 907                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 908                              |(?:                                             # or the v= param in all its forms
 909                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 910                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 911                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 912                                  v=
 913                              )
 914                          ))
 915                          |(?:
 916                             youtu\.be|                                        # just youtu.be/xxxx
 917                             vid\.plus|                                        # or vid.plus/xxxx
 918                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 919                             %(invidious)s
 920                          )/
 921                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 922                          )
 923                      )?                                                       # all until now is optional -> you can pass the naked ID
 924                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 925                      (?(1).+)?                                                # if we found the ID, everything can follow
 926                      (?:\#|$)""" % {
 927         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 928     }
 929     _PLAYER_INFO_RE = (
 930         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 931         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 932         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 933     )
 934     _formats = {
 935         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 936         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 937         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 938         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 939         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 940         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 941         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 942         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 943         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 944         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 945         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 946         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 947         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 948         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 949         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 950         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 951         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 952         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 953
 954
 955         # 3D videos
 956         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 957         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 958         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 959         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 960         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 961         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 962         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 963
 964         # Apple HTTP Live Streaming
 965         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 966         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 967         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 968         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 969         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 970         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 971         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 972         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 973
 974         # DASH mp4 video
 975         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 976         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 977         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 978         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 979         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 980         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 981         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 982         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 983         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 984         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 985         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 986         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 987
 988         # Dash mp4 audio
 989         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 990         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 991         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 992         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 993         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 994         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 995         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 996
 997         # Dash webm
 998         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 999         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1000         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1001         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1002         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1003         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1004         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1005         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1006         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1007         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1008         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1009         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1010         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1011         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1012         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1013         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1014         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1015         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1016         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1017         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1018         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1019         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1020
1021         # Dash webm audio
1022         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1023         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1024
1025         # Dash webm audio with opus inside
1026         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1027         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1028         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1029
1030         # RTMP (unnamed)
1031         '_rtmp': {'protocol': 'rtmp'},
1032
1033         # av01 video only formats sometimes served with "unknown" codecs
1034         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1035         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1036         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1037         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1038         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1039         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1040         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1041         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1042     }
1043     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1044
1045     _GEO_BYPASS = False
1046
1047     IE_NAME = 'youtube'
1048     _TESTS = [
1049         {
1050             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1051             'info_dict': {
1052                 'id': 'BaW_jenozKc',
1053                 'ext': 'mp4',
1054                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1055                 'uploader': 'Philipp Hagemeister',
1056                 'uploader_id': 'phihag',
1057                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1058                 'channel': 'Philipp Hagemeister',
1059                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1060                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1061                 'upload_date': '20121002',
1062                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1063                 'categories': ['Science & Technology'],
1064                 'tags': ['youtube-dl'],
1065                 'duration': 10,
1066                 'view_count': int,
1067                 'like_count': int,
1068                 'availability': 'public',
1069                 'playable_in_embed': True,
1070                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1071                 'live_status': 'not_live',
1072                 'age_limit': 0,
1073                 'start_time': 1,
1074                 'end_time': 9,
1075                 'channel_follower_count': int
1076             }
1077         },
1078         {
1079             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1080             'note': 'Embed-only video (#1746)',
1081             'info_dict': {
1082                 'id': 'yZIXLfi8CZQ',
1083                 'ext': 'mp4',
1084                 'upload_date': '20120608',
1085                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1086                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1087                 'uploader': 'SET India',
1088                 'uploader_id': 'setindia',
1089                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1090                 'age_limit': 18,
1091             },
1092             'skip': 'Private video',
1093         },
1094         {
1095             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1096             'note': 'Use the first video ID in the URL',
1097             'info_dict': {
1098                 'id': 'BaW_jenozKc',
1099                 'ext': 'mp4',
1100                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1101                 'uploader': 'Philipp Hagemeister',
1102                 'uploader_id': 'phihag',
1103                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1104                 'channel': 'Philipp Hagemeister',
1105                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1106                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1107                 'upload_date': '20121002',
1108                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1109                 'categories': ['Science & Technology'],
1110                 'tags': ['youtube-dl'],
1111                 'duration': 10,
1112                 'view_count': int,
1113                 'like_count': int,
1114                 'availability': 'public',
1115                 'playable_in_embed': True,
1116                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1117                 'live_status': 'not_live',
1118                 'age_limit': 0,
1119                 'channel_follower_count': int
1120             },
1121             'params': {
1122                 'skip_download': True,
1123             },
1124         },
1125         {
1126             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1127             'note': '256k DASH audio (format 141) via DASH manifest',
1128             'info_dict': {
1129                 'id': 'a9LDPn-MO4I',
1130                 'ext': 'm4a',
1131                 'upload_date': '20121002',
1132                 'uploader_id': '8KVIDEO',
1133                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1134                 'description': '',
1135                 'uploader': '8KVIDEO',
1136                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1137             },
1138             'params': {
1139                 'youtube_include_dash_manifest': True,
1140                 'format': '141',
1141             },
1142             'skip': 'format 141 not served anymore',
1143         },
1144         # DASH manifest with encrypted signature
1145         {
1146             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1147             'info_dict': {
1148                 'id': 'IB3lcPjvWLA',
1149                 'ext': 'm4a',
1150                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1151                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1152                 'duration': 244,
1153                 'uploader': 'AfrojackVEVO',
1154                 'uploader_id': 'AfrojackVEVO',
1155                 'upload_date': '20131011',
1156                 'abr': 129.495,
1157                 'like_count': int,
1158                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1159                 'playable_in_embed': True,
1160                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1161                 'view_count': int,
1162                 'track': 'The Spark',
1163                 'live_status': 'not_live',
1164                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1165                 'channel': 'Afrojack',
1166                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1167                 'tags': 'count:19',
1168                 'availability': 'public',
1169                 'categories': ['Music'],
1170                 'age_limit': 0,
1171                 'alt_title': 'The Spark',
1172                 'channel_follower_count': int
1173             },
1174             'params': {
1175                 'youtube_include_dash_manifest': True,
1176                 'format': '141/bestaudio[ext=m4a]',
1177             },
1178         },
1179         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1180         {
1181             'note': 'Embed allowed age-gate video',
1182             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1183             'info_dict': {
1184                 'id': 'HtVdAasjOgU',
1185                 'ext': 'mp4',
1186                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1187                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1188                 'duration': 142,
1189                 'uploader': 'The Witcher',
1190                 'uploader_id': 'WitcherGame',
1191                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1192                 'upload_date': '20140605',
1193                 'age_limit': 18,
1194                 'categories': ['Gaming'],
1195                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1196                 'availability': 'needs_auth',
1197                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1198                 'like_count': int,
1199                 'channel': 'The Witcher',
1200                 'live_status': 'not_live',
1201                 'tags': 'count:17',
1202                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1203                 'playable_in_embed': True,
1204                 'view_count': int,
1205                 'channel_follower_count': int
1206             },
1207         },
1208         {
1209             'note': 'Age-gate video with embed allowed in public site',
1210             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1211             'info_dict': {
1212                 'id': 'HsUATh_Nc2U',
1213                 'ext': 'mp4',
1214                 'title': 'Godzilla 2 (Official Video)',
1215                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1216                 'upload_date': '20200408',
1217                 'uploader_id': 'FlyingKitty900',
1218                 'uploader': 'FlyingKitty',
1219                 'age_limit': 18,
1220                 'availability': 'needs_auth',
1221                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1222                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1223                 'channel': 'FlyingKitty',
1224                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1225                 'view_count': int,
1226                 'categories': ['Entertainment'],
1227                 'live_status': 'not_live',
1228                 'tags': ['Flyingkitty', 'godzilla 2'],
1229                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1230                 'like_count': int,
1231                 'duration': 177,
1232                 'playable_in_embed': True,
1233                 'channel_follower_count': int
1234             },
1235         },
1236         {
1237             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1238             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1239             'info_dict': {
1240                 'id': 'Tq92D6wQ1mg',
1241                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1242                 'ext': 'mp4',
1243                 'upload_date': '20191228',
1244                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1245                 'uploader': 'Projekt Melody',
1246                 'description': 'md5:17eccca93a786d51bc67646756894066',
1247                 'age_limit': 18,
1248                 'like_count': int,
1249                 'availability': 'needs_auth',
1250                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1251                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1252                 'view_count': int,
1253                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1254                 'channel': 'Projekt Melody',
1255                 'live_status': 'not_live',
1256                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1257                 'playable_in_embed': True,
1258                 'categories': ['Entertainment'],
1259                 'duration': 106,
1260                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1261                 'channel_follower_count': int
1262             },
1263         },
1264         {
1265             'note': 'Non-Agegated non-embeddable video',
1266             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1267             'info_dict': {
1268                 'id': 'MeJVWBSsPAY',
1269                 'ext': 'mp4',
1270                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1271                 'uploader': 'Herr Lurik',
1272                 'uploader_id': 'st3in234',
1273                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1274                 'upload_date': '20130730',
1275                 'track': 'Such mich find mich',
1276                 'age_limit': 0,
1277                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1278                 'like_count': int,
1279                 'playable_in_embed': False,
1280                 'creator': 'OOMPH!',
1281                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1282                 'view_count': int,
1283                 'alt_title': 'Such mich find mich',
1284                 'duration': 210,
1285                 'channel': 'Herr Lurik',
1286                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1287                 'categories': ['Music'],
1288                 'availability': 'public',
1289                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1290                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1291                 'live_status': 'not_live',
1292                 'artist': 'OOMPH!',
1293                 'channel_follower_count': int
1294             },
1295         },
1296         {
1297             'note': 'Non-bypassable age-gated video',
1298             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1299             'only_matching': True,
1300         },
1301         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1302         # YouTube Red ad is not captured for creator
1303         {
1304             'url': '__2ABJjxzNo',
1305             'info_dict': {
1306                 'id': '__2ABJjxzNo',
1307                 'ext': 'mp4',
1308                 'duration': 266,
1309                 'upload_date': '20100430',
1310                 'uploader_id': 'deadmau5',
1311                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1312                 'creator': 'deadmau5',
1313                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1314                 'uploader': 'deadmau5',
1315                 'title': 'Deadmau5 - Some Chords (HD)',
1316                 'alt_title': 'Some Chords',
1317                 'availability': 'public',
1318                 'tags': 'count:14',
1319                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1320                 'view_count': int,
1321                 'live_status': 'not_live',
1322                 'channel': 'deadmau5',
1323                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1324                 'like_count': int,
1325                 'track': 'Some Chords',
1326                 'artist': 'deadmau5',
1327                 'playable_in_embed': True,
1328                 'age_limit': 0,
1329                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1330                 'categories': ['Music'],
1331                 'album': 'Some Chords',
1332                 'channel_follower_count': int
1333             },
1334             'expected_warnings': [
1335                 'DASH manifest missing',
1336             ]
1337         },
1338         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1339         {
1340             'url': 'lqQg6PlCWgI',
1341             'info_dict': {
1342                 'id': 'lqQg6PlCWgI',
1343                 'ext': 'mp4',
1344                 'duration': 6085,
1345                 'upload_date': '20150827',
1346                 'uploader_id': 'olympic',
1347                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1348                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1349                 'uploader': 'Olympics',
1350                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1351                 'like_count': int,
1352                 'release_timestamp': 1343767800,
1353                 'playable_in_embed': True,
1354                 'categories': ['Sports'],
1355                 'release_date': '20120731',
1356                 'channel': 'Olympics',
1357                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1358                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1359                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1360                 'age_limit': 0,
1361                 'availability': 'public',
1362                 'live_status': 'was_live',
1363                 'view_count': int,
1364                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1365                 'channel_follower_count': int
1366             },
1367             'params': {
1368                 'skip_download': 'requires avconv',
1369             }
1370         },
1371         # Non-square pixels
1372         {
1373             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1374             'info_dict': {
1375                 'id': '_b-2C3KPAM0',
1376                 'ext': 'mp4',
1377                 'stretched_ratio': 16 / 9.,
1378                 'duration': 85,
1379                 'upload_date': '20110310',
1380                 'uploader_id': 'AllenMeow',
1381                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1382                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1383                 'uploader': '孫ᄋᄅ',
1384                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1385                 'playable_in_embed': True,
1386                 'channel': '孫ᄋᄅ',
1387                 'age_limit': 0,
1388                 'tags': 'count:11',
1389                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1390                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1391                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1392                 'view_count': int,
1393                 'categories': ['People & Blogs'],
1394                 'like_count': int,
1395                 'live_status': 'not_live',
1396                 'availability': 'unlisted',
1397                 'channel_follower_count': int
1398             },
1399         },
1400         # url_encoded_fmt_stream_map is empty string
1401         {
1402             'url': 'qEJwOuvDf7I',
1403             'info_dict': {
1404                 'id': 'qEJwOuvDf7I',
1405                 'ext': 'webm',
1406                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1407                 'description': '',
1408                 'upload_date': '20150404',
1409                 'uploader_id': 'spbelect',
1410                 'uploader': 'Наблюдатели Петербурга',
1411             },
1412             'params': {
1413                 'skip_download': 'requires avconv',
1414             },
1415             'skip': 'This live event has ended.',
1416         },
1417         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1418         {
1419             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1420             'info_dict': {
1421                 'id': 'FIl7x6_3R5Y',
1422                 'ext': 'webm',
1423                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1424                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1425                 'duration': 220,
1426                 'upload_date': '20150625',
1427                 'uploader_id': 'dorappi2000',
1428                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1429                 'uploader': 'dorappi2000',
1430                 'formats': 'mincount:31',
1431             },
1432             'skip': 'not actual anymore',
1433         },
1434         # DASH manifest with segment_list
1435         {
1436             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1437             'md5': '8ce563a1d667b599d21064e982ab9e31',
1438             'info_dict': {
1439                 'id': 'CsmdDsKjzN8',
1440                 'ext': 'mp4',
1441                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1442                 'uploader': 'Airtek',
1443                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1444                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1445                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1446             },
1447             'params': {
1448                 'youtube_include_dash_manifest': True,
1449                 'format': '135',  # bestvideo
1450             },
1451             'skip': 'This live event has ended.',
1452         },
1453         {
1454             # Multifeed videos (multiple cameras), URL is for Main Camera
1455             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1456             'info_dict': {
1457                 'id': 'jvGDaLqkpTg',
1458                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1459                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1460             },
1461             'playlist': [{
1462                 'info_dict': {
1463                     'id': 'jvGDaLqkpTg',
1464                     'ext': 'mp4',
1465                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1466                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1467                     'duration': 10643,
1468                     'upload_date': '20161111',
1469                     'uploader': 'Team PGP',
1470                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1471                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1472                 },
1473             }, {
1474                 'info_dict': {
1475                     'id': '3AKt1R1aDnw',
1476                     'ext': 'mp4',
1477                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1478                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1479                     'duration': 10991,
1480                     'upload_date': '20161111',
1481                     'uploader': 'Team PGP',
1482                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1483                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1484                 },
1485             }, {
1486                 'info_dict': {
1487                     'id': 'RtAMM00gpVc',
1488                     'ext': 'mp4',
1489                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1490                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1491                     'duration': 10995,
1492                     'upload_date': '20161111',
1493                     'uploader': 'Team PGP',
1494                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1495                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1496                 },
1497             }, {
1498                 'info_dict': {
1499                     'id': '6N2fdlP3C5U',
1500                     'ext': 'mp4',
1501                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1502                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1503                     'duration': 10990,
1504                     'upload_date': '20161111',
1505                     'uploader': 'Team PGP',
1506                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1507                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1508                 },
1509             }],
1510             'params': {
1511                 'skip_download': True,
1512             },
1513             'skip': 'Not multifeed anymore',
1514         },
1515         {
1516             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1517             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1518             'info_dict': {
1519                 'id': 'gVfLd0zydlo',
1520                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1521             },
1522             'playlist_count': 2,
1523             'skip': 'Not multifeed anymore',
1524         },
1525         {
1526             'url': 'https://vid.plus/FlRa-iH7PGw',
1527             'only_matching': True,
1528         },
1529         {
1530             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1531             'only_matching': True,
1532         },
1533         {
1534             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1535             # Also tests cut-off URL expansion in video description (see
1536             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1537             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1538             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1539             'info_dict': {
1540                 'id': 'lsguqyKfVQg',
1541                 'ext': 'mp4',
1542                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1543                 'alt_title': 'Dark Walk',
1544                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1545                 'duration': 133,
1546                 'upload_date': '20151119',
1547                 'uploader_id': 'IronSoulElf',
1548                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1549                 'uploader': 'IronSoulElf',
1550                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1551                 'track': 'Dark Walk',
1552                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1553                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1554                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1555                 'categories': ['Film & Animation'],
1556                 'view_count': int,
1557                 'live_status': 'not_live',
1558                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1559                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1560                 'tags': 'count:13',
1561                 'availability': 'public',
1562                 'channel': 'IronSoulElf',
1563                 'playable_in_embed': True,
1564                 'like_count': int,
1565                 'age_limit': 0,
1566                 'channel_follower_count': int
1567             },
1568             'params': {
1569                 'skip_download': True,
1570             },
1571         },
1572         {
1573             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1574             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1575             'only_matching': True,
1576         },
1577         {
1578             # Video with yt:stretch=17:0
1579             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1580             'info_dict': {
1581                 'id': 'Q39EVAstoRM',
1582                 'ext': 'mp4',
1583                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1584                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1585                 'upload_date': '20151107',
1586                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1587                 'uploader': 'CH GAMER DROID',
1588             },
1589             'params': {
1590                 'skip_download': True,
1591             },
1592             'skip': 'This video does not exist.',
1593         },
1594         {
1595             # Video with incomplete 'yt:stretch=16:'
1596             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1597             'only_matching': True,
1598         },
1599         {
1600             # Video licensed under Creative Commons
1601             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1602             'info_dict': {
1603                 'id': 'M4gD1WSo5mA',
1604                 'ext': 'mp4',
1605                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1606                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1607                 'duration': 721,
1608                 'upload_date': '20150128',
1609                 'uploader_id': 'BerkmanCenter',
1610                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1611                 'uploader': 'The Berkman Klein Center for Internet & Society',
1612                 'license': 'Creative Commons Attribution license (reuse allowed)',
1613                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1614                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1615                 'like_count': int,
1616                 'age_limit': 0,
1617                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1618                 'channel': 'The Berkman Klein Center for Internet & Society',
1619                 'availability': 'public',
1620                 'view_count': int,
1621                 'categories': ['Education'],
1622                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1623                 'live_status': 'not_live',
1624                 'playable_in_embed': True,
1625                 'channel_follower_count': int
1626             },
1627             'params': {
1628                 'skip_download': True,
1629             },
1630         },
1631         {
1632             # Channel-like uploader_url
1633             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1634             'info_dict': {
1635                 'id': 'eQcmzGIKrzg',
1636                 'ext': 'mp4',
1637                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1638                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1639                 'duration': 4060,
1640                 'upload_date': '20151120',
1641                 'uploader': 'Bernie Sanders',
1642                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1643                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1644                 'license': 'Creative Commons Attribution license (reuse allowed)',
1645                 'playable_in_embed': True,
1646                 'tags': 'count:12',
1647                 'like_count': int,
1648                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1649                 'age_limit': 0,
1650                 'availability': 'public',
1651                 'categories': ['News & Politics'],
1652                 'channel': 'Bernie Sanders',
1653                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1654                 'view_count': int,
1655                 'live_status': 'not_live',
1656                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1657                 'channel_follower_count': int
1658             },
1659             'params': {
1660                 'skip_download': True,
1661             },
1662         },
1663         {
1664             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1665             'only_matching': True,
1666         },
1667         {
1668             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1669             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1670             'only_matching': True,
1671         },
1672         {
1673             # Rental video preview
1674             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1675             'info_dict': {
1676                 'id': 'uGpuVWrhIzE',
1677                 'ext': 'mp4',
1678                 'title': 'Piku - Trailer',
1679                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1680                 'upload_date': '20150811',
1681                 'uploader': 'FlixMatrix',
1682                 'uploader_id': 'FlixMatrixKaravan',
1683                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1684                 'license': 'Standard YouTube License',
1685             },
1686             'params': {
1687                 'skip_download': True,
1688             },
1689             'skip': 'This video is not available.',
1690         },
1691         {
1692             # YouTube Red video with episode data
1693             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1694             'info_dict': {
1695                 'id': 'iqKdEhx-dD4',
1696                 'ext': 'mp4',
1697                 'title': 'Isolation - Mind Field (Ep 1)',
1698                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1699                 'duration': 2085,
1700                 'upload_date': '20170118',
1701                 'uploader': 'Vsauce',
1702                 'uploader_id': 'Vsauce',
1703                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1704                 'series': 'Mind Field',
1705                 'season_number': 1,
1706                 'episode_number': 1,
1707                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1708                 'tags': 'count:12',
1709                 'view_count': int,
1710                 'availability': 'public',
1711                 'age_limit': 0,
1712                 'channel': 'Vsauce',
1713                 'episode': 'Episode 1',
1714                 'categories': ['Entertainment'],
1715                 'season': 'Season 1',
1716                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1717                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1718                 'like_count': int,
1719                 'playable_in_embed': True,
1720                 'live_status': 'not_live',
1721                 'channel_follower_count': int
1722             },
1723             'params': {
1724                 'skip_download': True,
1725             },
1726             'expected_warnings': [
1727                 'Skipping DASH manifest',
1728             ],
1729         },
1730         {
1731             # The following content has been identified by the YouTube community
1732             # as inappropriate or offensive to some audiences.
1733             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1734             'info_dict': {
1735                 'id': '6SJNVb0GnPI',
1736                 'ext': 'mp4',
1737                 'title': 'Race Differences in Intelligence',
1738                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1739                 'duration': 965,
1740                 'upload_date': '20140124',
1741                 'uploader': 'New Century Foundation',
1742                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1743                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1744             },
1745             'params': {
1746                 'skip_download': True,
1747             },
1748             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1749         },
1750         {
1751             # itag 212
1752             'url': '1t24XAntNCY',
1753             'only_matching': True,
1754         },
1755         {
1756             # geo restricted to JP
1757             'url': 'sJL6WA-aGkQ',
1758             'only_matching': True,
1759         },
1760         {
1761             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1762             'only_matching': True,
1763         },
1764         {
1765             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1766             'only_matching': True,
1767         },
1768         {
1769             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1770             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1771             'only_matching': True,
1772         },
1773         {
1774             # DRM protected
1775             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1776             'only_matching': True,
1777         },
1778         {
1779             # Video with unsupported adaptive stream type formats
1780             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1781             'info_dict': {
1782                 'id': 'Z4Vy8R84T1U',
1783                 'ext': 'mp4',
1784                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1785                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1786                 'duration': 433,
1787                 'upload_date': '20130923',
1788                 'uploader': 'Amelia Putri Harwita',
1789                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1790                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1791                 'formats': 'maxcount:10',
1792             },
1793             'params': {
1794                 'skip_download': True,
1795                 'youtube_include_dash_manifest': False,
1796             },
1797             'skip': 'not actual anymore',
1798         },
1799         {
1800             # Youtube Music Auto-generated description
1801             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1802             'info_dict': {
1803                 'id': 'MgNrAu2pzNs',
1804                 'ext': 'mp4',
1805                 'title': 'Voyeur Girl',
1806                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1807                 'upload_date': '20190312',
1808                 'uploader': 'Stephen - Topic',
1809                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1810                 'artist': 'Stephen',
1811                 'track': 'Voyeur Girl',
1812                 'album': 'it\'s too much love to know my dear',
1813                 'release_date': '20190313',
1814                 'release_year': 2019,
1815                 'alt_title': 'Voyeur Girl',
1816                 'view_count': int,
1817                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1818                 'playable_in_embed': True,
1819                 'like_count': int,
1820                 'categories': ['Music'],
1821                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1822                 'channel': 'Stephen',
1823                 'availability': 'public',
1824                 'creator': 'Stephen',
1825                 'duration': 169,
1826                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1827                 'age_limit': 0,
1828                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1829                 'tags': 'count:11',
1830                 'live_status': 'not_live',
1831                 'channel_follower_count': int
1832             },
1833             'params': {
1834                 'skip_download': True,
1835             },
1836         },
1837         {
1838             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1839             'only_matching': True,
1840         },
1841         {
1842             # invalid -> valid video id redirection
1843             'url': 'DJztXj2GPfl',
1844             'info_dict': {
1845                 'id': 'DJztXj2GPfk',
1846                 'ext': 'mp4',
1847                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1848                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1849                 'upload_date': '20090125',
1850                 'uploader': 'Prochorowka',
1851                 'uploader_id': 'Prochorowka',
1852                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1853                 'artist': 'Panjabi MC',
1854                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1855                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1856             },
1857             'params': {
1858                 'skip_download': True,
1859             },
1860             'skip': 'Video unavailable',
1861         },
1862         {
1863             # empty description results in an empty string
1864             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1865             'info_dict': {
1866                 'id': 'x41yOUIvK2k',
1867                 'ext': 'mp4',
1868                 'title': 'IMG 3456',
1869                 'description': '',
1870                 'upload_date': '20170613',
1871                 'uploader_id': 'ElevageOrVert',
1872                 'uploader': 'ElevageOrVert',
1873                 'view_count': int,
1874                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1875                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1876                 'like_count': int,
1877                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1878                 'tags': [],
1879                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1880                 'availability': 'public',
1881                 'age_limit': 0,
1882                 'categories': ['Pets & Animals'],
1883                 'duration': 7,
1884                 'playable_in_embed': True,
1885                 'live_status': 'not_live',
1886                 'channel': 'ElevageOrVert',
1887                 'channel_follower_count': int
1888             },
1889             'params': {
1890                 'skip_download': True,
1891             },
1892         },
1893         {
1894             # with '};' inside yt initial data (see [1])
1895             # see [2] for an example with '};' inside ytInitialPlayerResponse
1896             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1897             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1898             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1899             'info_dict': {
1900                 'id': 'CHqg6qOn4no',
1901                 'ext': 'mp4',
1902                 'title': 'Part 77   Sort a list of simple types in c#',
1903                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1904                 'upload_date': '20130831',
1905                 'uploader_id': 'kudvenkat',
1906                 'uploader': 'kudvenkat',
1907                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1908                 'like_count': int,
1909                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1910                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1911                 'live_status': 'not_live',
1912                 'categories': ['Education'],
1913                 'availability': 'public',
1914                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1915                 'tags': 'count:12',
1916                 'playable_in_embed': True,
1917                 'age_limit': 0,
1918                 'view_count': int,
1919                 'duration': 522,
1920                 'channel': 'kudvenkat',
1921                 'channel_follower_count': int
1922             },
1923             'params': {
1924                 'skip_download': True,
1925             },
1926         },
1927         {
1928             # another example of '};' in ytInitialData
1929             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1930             'only_matching': True,
1931         },
1932         {
1933             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1934             'only_matching': True,
1935         },
1936         {
1937             # https://github.com/ytdl-org/youtube-dl/pull/28094
1938             'url': 'OtqTfy26tG0',
1939             'info_dict': {
1940                 'id': 'OtqTfy26tG0',
1941                 'ext': 'mp4',
1942                 'title': 'Burn Out',
1943                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1944                 'upload_date': '20141120',
1945                 'uploader': 'The Cinematic Orchestra - Topic',
1946                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1947                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1948                 'artist': 'The Cinematic Orchestra',
1949                 'track': 'Burn Out',
1950                 'album': 'Every Day',
1951                 'like_count': int,
1952                 'live_status': 'not_live',
1953                 'alt_title': 'Burn Out',
1954                 'duration': 614,
1955                 'age_limit': 0,
1956                 'view_count': int,
1957                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1958                 'creator': 'The Cinematic Orchestra',
1959                 'channel': 'The Cinematic Orchestra',
1960                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1961                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1962                 'availability': 'public',
1963                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1964                 'categories': ['Music'],
1965                 'playable_in_embed': True,
1966                 'channel_follower_count': int
1967             },
1968             'params': {
1969                 'skip_download': True,
1970             },
1971         },
1972         {
1973             # controversial video, only works with bpctr when authenticated with cookies
1974             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1975             'only_matching': True,
1976         },
1977         {
1978             # controversial video, requires bpctr/contentCheckOk
1979             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1980             'info_dict': {
1981                 'id': 'SZJvDhaSDnc',
1982                 'ext': 'mp4',
1983                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1984                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1985                 'uploader': 'CBS Mornings',
1986                 'uploader_id': 'CBSThisMorning',
1987                 'upload_date': '20140716',
1988                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1989                 'duration': 170,
1990                 'categories': ['News & Politics'],
1991                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
1992                 'view_count': int,
1993                 'channel': 'CBS Mornings',
1994                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
1995                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
1996                 'age_limit': 18,
1997                 'availability': 'needs_auth',
1998                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
1999                 'like_count': int,
2000                 'live_status': 'not_live',
2001                 'playable_in_embed': True,
2002                 'channel_follower_count': int
2003             }
2004         },
2005         {
2006             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2007             'url': 'cBvYw8_A0vQ',
2008             'info_dict': {
2009                 'id': 'cBvYw8_A0vQ',
2010                 'ext': 'mp4',
2011                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2012                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2013                 'upload_date': '20201120',
2014                 'uploader': 'Walk around Japan',
2015                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2016                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2017                 'duration': 1456,
2018                 'categories': ['Travel & Events'],
2019                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2020                 'view_count': int,
2021                 'channel': 'Walk around Japan',
2022                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2023                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2024                 'age_limit': 0,
2025                 'availability': 'public',
2026                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2027                 'live_status': 'not_live',
2028                 'playable_in_embed': True,
2029                 'channel_follower_count': int
2030             },
2031             'params': {
2032                 'skip_download': True,
2033             },
2034         }, {
2035             # Has multiple audio streams
2036             'url': 'WaOKSUlf4TM',
2037             'only_matching': True
2038         }, {
2039             # Requires Premium: has format 141 when requested using YTM url
2040             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2041             'only_matching': True
2042         }, {
2043             # multiple subtitles with same lang_code
2044             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2045             'only_matching': True,
2046         }, {
2047             # Force use android client fallback
2048             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2049             'info_dict': {
2050                 'id': 'YOelRv7fMxY',
2051                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2052                 'ext': '3gp',
2053                 'upload_date': '20210624',
2054                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2055                 'uploader': 'colinfurze',
2056                 'uploader_id': 'colinfurze',
2057                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2058                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2059                 'duration': 596,
2060                 'categories': ['Entertainment'],
2061                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2062                 'view_count': int,
2063                 'channel': 'colinfurze',
2064                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2065                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2066                 'age_limit': 0,
2067                 'availability': 'public',
2068                 'like_count': int,
2069                 'live_status': 'not_live',
2070                 'playable_in_embed': True,
2071                 'channel_follower_count': int
2072             },
2073             'params': {
2074                 'format': '17',  # 3gp format available on android
2075                 'extractor_args': {'youtube': {'player_client': ['android']}},
2076             },
2077         },
2078         {
2079             # Skip download of additional client configs (remix client config in this case)
2080             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2081             'only_matching': True,
2082             'params': {
2083                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2084             },
2085         }, {
2086             # shorts
2087             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2088             'only_matching': True,
2089         }, {
2090             'note': 'Storyboards',
2091             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2092             'info_dict': {
2093                 'id': '5KLPxDtMqe8',
2094                 'ext': 'mhtml',
2095                 'format_id': 'sb0',
2096                 'title': 'Your Brain is Plastic',
2097                 'uploader_id': 'scishow',
2098                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2099                 'upload_date': '20140324',
2100                 'uploader': 'SciShow',
2101                 'like_count': int,
2102                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2103                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2104                 'view_count': int,
2105                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2106                 'playable_in_embed': True,
2107                 'tags': 'count:12',
2108                 'uploader_url': 'http://www.youtube.com/user/scishow',
2109                 'availability': 'public',
2110                 'channel': 'SciShow',
2111                 'live_status': 'not_live',
2112                 'duration': 248,
2113                 'categories': ['Education'],
2114                 'age_limit': 0,
2115                 'channel_follower_count': int
2116             }, 'params': {'format': 'mhtml', 'skip_download': True}
2117         }, {
2118             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2119             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2120             'info_dict': {
2121                 'id': '2NUZ8W2llS4',
2122                 'ext': 'mp4',
2123                 'title': 'The NP that test your phone performance 🙂',
2124                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2125                 'uploader': 'Leon Nguyen',
2126                 'uploader_id': 'VNSXIII',
2127                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2128                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2129                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2130                 'duration': 21,
2131                 'view_count': int,
2132                 'age_limit': 0,
2133                 'categories': ['Gaming'],
2134                 'tags': 'count:23',
2135                 'playable_in_embed': True,
2136                 'live_status': 'not_live',
2137                 'upload_date': '20220103',
2138                 'like_count': int,
2139                 'availability': 'public',
2140                 'channel': 'Leon Nguyen',
2141                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2142                 'channel_follower_count': int
2143             }
2144         }, {
2145             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2146             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2147             'info_dict': {
2148                 'id': 'mzZzzBU6lrM',
2149                 'ext': 'mp4',
2150                 'title': 'I Met GeorgeNotFound In Real Life...',
2151                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2152                 'uploader': 'Quackity',
2153                 'uploader_id': 'QuackityHQ',
2154                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2155                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2156                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2157                 'duration': 955,
2158                 'view_count': int,
2159                 'age_limit': 0,
2160                 'categories': ['Entertainment'],
2161                 'tags': 'count:26',
2162                 'playable_in_embed': True,
2163                 'live_status': 'not_live',
2164                 'release_timestamp': 1641172509,
2165                 'release_date': '20220103',
2166                 'upload_date': '20220103',
2167                 'like_count': int,
2168                 'availability': 'public',
2169                 'channel': 'Quackity',
2170                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2171                 'channel_follower_count': int
2172             }
2173         },
2174         {   # continuous livestream. Microformat upload date should be preferred.
2175             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2176             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2177             'info_dict': {
2178                 'id': 'kgx4WGK0oNU',
2179                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2180                 'ext': 'mp4',
2181                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2182                 'availability': 'public',
2183                 'age_limit': 0,
2184                 'release_timestamp': 1637975704,
2185                 'upload_date': '20210619',
2186                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2187                 'live_status': 'is_live',
2188                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2189                 'uploader': '阿鲍Abao',
2190                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2191                 'channel': 'Abao in Tokyo',
2192                 'channel_follower_count': int,
2193                 'release_date': '20211127',
2194                 'tags': 'count:39',
2195                 'categories': ['People & Blogs'],
2196                 'like_count': int,
2197                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2198                 'view_count': int,
2199                 'playable_in_embed': True,
2200                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2201             },
2202             'params': {'skip_download': True}
2203         },
2204     ]
2205
2206     @classmethod
2207     def suitable(cls, url):
2208         from ..utils import parse_qs
2209
2210         qs = parse_qs(url)
2211         if qs.get('list', [None])[0]:
2212             return False
2213         return super().suitable(url)
2214
2215     def __init__(self, *args, **kwargs):
2216         super().__init__(*args, **kwargs)
2217         self._code_cache = {}
2218         self._player_cache = {}
2219
2220     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2221         lock = threading.Lock()
2222
2223         is_live = True
2224         start_time = time.time()
2225         formats = [f for f in formats if f.get('is_from_start')]
2226
2227         def refetch_manifest(format_id, delay):
2228             nonlocal formats, start_time, is_live
2229             if time.time() <= start_time + delay:
2230                 return
2231
2232             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2233             video_details = traverse_obj(
2234                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2235             microformats = traverse_obj(
2236                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2237                 expected_type=dict, default=[])
2238             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2239             start_time = time.time()
2240
2241         def mpd_feed(format_id, delay):
2242             """
2243             @returns (manifest_url, manifest_stream_number, is_live) or None
2244             """
2245             with lock:
2246                 refetch_manifest(format_id, delay)
2247
2248             f = next((f for f in formats if f['format_id'] == format_id), None)
2249             if not f:
2250                 if not is_live:
2251                     self.to_screen(f'{video_id}: Video is no longer live')
2252                 else:
2253                     self.report_warning(
2254                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2255                 return None
2256             return f['manifest_url'], f['manifest_stream_number'], is_live
2257
2258         for f in formats:
2259             f['is_live'] = True
2260             f['protocol'] = 'http_dash_segments_generator'
2261             f['fragments'] = functools.partial(
2262                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2263
2264     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2265         FETCH_SPAN, MAX_DURATION = 5, 432000
2266
2267         mpd_url, stream_number, is_live = None, None, True
2268
2269         begin_index = 0
2270         download_start_time = ctx.get('start') or time.time()
2271
2272         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2273         if lack_early_segments:
2274             self.report_warning(bug_reports_message(
2275                 'Starting download from the last 120 hours of the live stream since '
2276                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2277             lack_early_segments = True
2278
2279         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2280         fragments, fragment_base_url = None, None
2281
2282         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2283             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2284             # Obtain from MPD's maximum seq value
2285             old_mpd_url = mpd_url
2286             last_error = ctx.pop('last_error', None)
2287             expire_fast = immediate or last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2288             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2289                                                or (mpd_url, stream_number, False))
2290             if not refresh_sequence:
2291                 if expire_fast and not is_live:
2292                     return False, last_seq
2293                 elif old_mpd_url == mpd_url:
2294                     return True, last_seq
2295             try:
2296                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2297                     mpd_url, None, note=False, errnote=False, fatal=False)
2298             except ExtractorError:
2299                 fmts = None
2300             if not fmts:
2301                 no_fragment_score += 2
2302                 return False, last_seq
2303             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2304             fragments = fmt_info['fragments']
2305             fragment_base_url = fmt_info['fragment_base_url']
2306             assert fragment_base_url
2307
2308             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2309             return True, _last_seq
2310
2311         while is_live:
2312             fetch_time = time.time()
2313             if no_fragment_score > 30:
2314                 return
2315             if last_segment_url:
2316                 # Obtain from "X-Head-Seqnum" header value from each segment
2317                 try:
2318                     urlh = self._request_webpage(
2319                         last_segment_url, None, note=False, errnote=False, fatal=False)
2320                 except ExtractorError:
2321                     urlh = None
2322                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2323                 if last_seq is None:
2324                     no_fragment_score += 2
2325                     last_segment_url = None
2326                     continue
2327             else:
2328                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2329                 no_fragment_score += 2
2330                 if not should_continue:
2331                     continue
2332
2333             if known_idx > last_seq:
2334                 last_segment_url = None
2335                 continue
2336
2337             last_seq += 1
2338
2339             if begin_index < 0 and known_idx < 0:
2340                 # skip from the start when it's negative value
2341                 known_idx = last_seq + begin_index
2342             if lack_early_segments:
2343                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2344             try:
2345                 for idx in range(known_idx, last_seq):
2346                     # do not update sequence here or you'll get skipped some part of it
2347                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2348                     if not should_continue:
2349                         known_idx = idx - 1
2350                         raise ExtractorError('breaking out of outer loop')
2351                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2352                     yield {
2353                         'url': last_segment_url,
2354                     }
2355                 if known_idx == last_seq:
2356                     no_fragment_score += 5
2357                 else:
2358                     no_fragment_score = 0
2359                 known_idx = last_seq
2360             except ExtractorError:
2361                 continue
2362
2363             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2364
2365     def _extract_player_url(self, *ytcfgs, webpage=None):
2366         player_url = traverse_obj(
2367             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2368             get_all=False, expected_type=compat_str)
2369         if not player_url:
2370             return
2371         return urljoin('https://www.youtube.com', player_url)
2372
2373     def _download_player_url(self, video_id, fatal=False):
2374         res = self._download_webpage(
2375             'https://www.youtube.com/iframe_api',
2376             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2377         if res:
2378             player_version = self._search_regex(
2379                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2380             if player_version:
2381                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2382
2383     def _signature_cache_id(self, example_sig):
2384         """ Return a string representation of a signature """
2385         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2386
2387     @classmethod
2388     def _extract_player_info(cls, player_url):
2389         for player_re in cls._PLAYER_INFO_RE:
2390             id_m = re.search(player_re, player_url)
2391             if id_m:
2392                 break
2393         else:
2394             raise ExtractorError('Cannot identify player %r' % player_url)
2395         return id_m.group('id')
2396
2397     def _load_player(self, video_id, player_url, fatal=True):
2398         player_id = self._extract_player_info(player_url)
2399         if player_id not in self._code_cache:
2400             code = self._download_webpage(
2401                 player_url, video_id, fatal=fatal,
2402                 note='Downloading player ' + player_id,
2403                 errnote='Download of %s failed' % player_url)
2404             if code:
2405                 self._code_cache[player_id] = code
2406         return self._code_cache.get(player_id)
2407
2408     def _extract_signature_function(self, video_id, player_url, example_sig):
2409         player_id = self._extract_player_info(player_url)
2410
2411         # Read from filesystem cache
2412         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2413         assert os.path.basename(func_id) == func_id
2414
2415         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
2416         if cache_spec is not None:
2417             return lambda s: ''.join(s[i] for i in cache_spec)
2418
2419         code = self._load_player(video_id, player_url)
2420         if code:
2421             res = self._parse_sig_js(code)
2422
2423             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2424             cache_res = res(test_string)
2425             cache_spec = [ord(c) for c in cache_res]
2426
2427             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
2428             return res
2429
2430     def _print_sig_code(self, func, example_sig):
2431         if not self.get_param('youtube_print_sig_code'):
2432             return
2433
2434         def gen_sig_code(idxs):
2435             def _genslice(start, end, step):
2436                 starts = '' if start == 0 else str(start)
2437                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2438                 steps = '' if step == 1 else (':%d' % step)
2439                 return f's[{starts}{ends}{steps}]'
2440
2441             step = None
2442             # Quelch pyflakes warnings - start will be set when step is set
2443             start = '(Never used)'
2444             for i, prev in zip(idxs[1:], idxs[:-1]):
2445                 if step is not None:
2446                     if i - prev == step:
2447                         continue
2448                     yield _genslice(start, prev, step)
2449                     step = None
2450                     continue
2451                 if i - prev in [-1, 1]:
2452                     step = i - prev
2453                     start = prev
2454                     continue
2455                 else:
2456                     yield 's[%d]' % prev
2457             if step is None:
2458                 yield 's[%d]' % i
2459             else:
2460                 yield _genslice(start, i, step)
2461
2462         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2463         cache_res = func(test_string)
2464         cache_spec = [ord(c) for c in cache_res]
2465         expr_code = ' + '.join(gen_sig_code(cache_spec))
2466         signature_id_tuple = '(%s)' % (
2467             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2468         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2469                 '    return %s\n') % (signature_id_tuple, expr_code)
2470         self.to_screen('Extracted signature function:\n' + code)
2471
2472     def _parse_sig_js(self, jscode):
2473         funcname = self._search_regex(
2474             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2475              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2476              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2477              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2478              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2479              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2480              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2481              # Obsolete patterns
2482              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2483              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2484              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2485              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2486              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2487              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2488              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2489              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2490             jscode, 'Initial JS player signature function name', group='sig')
2491
2492         jsi = JSInterpreter(jscode)
2493         initial_function = jsi.extract_function(funcname)
2494         return lambda s: initial_function([s])
2495
2496     def _decrypt_signature(self, s, video_id, player_url):
2497         """Turn the encrypted s field into a working signature"""
2498
2499         if player_url is None:
2500             raise ExtractorError('Cannot decrypt signature without player_url')
2501
2502         try:
2503             player_id = (player_url, self._signature_cache_id(s))
2504             if player_id not in self._player_cache:
2505                 func = self._extract_signature_function(
2506                     video_id, player_url, s
2507                 )
2508                 self._player_cache[player_id] = func
2509             func = self._player_cache[player_id]
2510             self._print_sig_code(func, s)
2511             return func(s)
2512         except Exception as e:
2513             raise ExtractorError('Signature extraction failed: ' + traceback.format_exc(), cause=e)
2514
2515     def _decrypt_nsig(self, s, video_id, player_url):
2516         """Turn the encrypted n field into a working signature"""
2517         if player_url is None:
2518             raise ExtractorError('Cannot decrypt nsig without player_url')
2519         player_url = urljoin('https://www.youtube.com', player_url)
2520
2521         sig_id = ('nsig_value', s)
2522         if sig_id in self._player_cache:
2523             return self._player_cache[sig_id]
2524
2525         try:
2526             player_id = ('nsig', player_url)
2527             if player_id not in self._player_cache:
2528                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2529             func = self._player_cache[player_id]
2530             self._player_cache[sig_id] = func(s)
2531             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2532             return self._player_cache[sig_id]
2533         except Exception as e:
2534             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2535
2536     def _extract_n_function_name(self, jscode):
2537         nfunc, idx = self._search_regex(
2538             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
2539             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
2540         if not idx:
2541             return nfunc
2542         return json.loads(js_to_json(self._search_regex(
2543             rf'var {re.escape(nfunc)}\s*=\s*(\[.+?\]);', jscode,
2544             f'Initial JS player n function list ({nfunc}.{idx})')))[int(idx)]
2545
2546     def _extract_n_function(self, video_id, player_url):
2547         player_id = self._extract_player_info(player_url)
2548         func_code = self._downloader.cache.load('youtube-nsig', player_id)
2549
2550         if func_code:
2551             jsi = JSInterpreter(func_code)
2552         else:
2553             jscode = self._load_player(video_id, player_url)
2554             funcname = self._extract_n_function_name(jscode)
2555             jsi = JSInterpreter(jscode)
2556             func_code = jsi.extract_function_code(funcname)
2557             self._downloader.cache.store('youtube-nsig', player_id, func_code)
2558
2559         if self.get_param('youtube_print_sig_code'):
2560             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2561
2562         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2563
2564     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2565         """
2566         Extract signatureTimestamp (sts)
2567         Required to tell API what sig/player version is in use.
2568         """
2569         sts = None
2570         if isinstance(ytcfg, dict):
2571             sts = int_or_none(ytcfg.get('STS'))
2572
2573         if not sts:
2574             # Attempt to extract from player
2575             if player_url is None:
2576                 error_msg = 'Cannot extract signature timestamp without player_url.'
2577                 if fatal:
2578                     raise ExtractorError(error_msg)
2579                 self.report_warning(error_msg)
2580                 return
2581             code = self._load_player(video_id, player_url, fatal=fatal)
2582             if code:
2583                 sts = int_or_none(self._search_regex(
2584                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2585                     'JS player signature timestamp', group='sts', fatal=fatal))
2586         return sts
2587
2588     def _mark_watched(self, video_id, player_responses):
2589         playback_url = get_first(
2590             player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2591             expected_type=url_or_none)
2592         if not playback_url:
2593             self.report_warning('Unable to mark watched')
2594             return
2595         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2596         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2597
2598         # cpn generation algorithm is reverse engineered from base.js.
2599         # In fact it works even with dummy cpn.
2600         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2601         cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
2602
2603         qs.update({
2604             'ver': ['2'],
2605             'cpn': [cpn],
2606         })
2607         playback_url = compat_urlparse.urlunparse(
2608             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2609
2610         self._download_webpage(
2611             playback_url, video_id, 'Marking watched',
2612             'Unable to mark watched', fatal=False)
2613
2614     @staticmethod
2615     def _extract_urls(webpage):
2616         # Embedded YouTube player
2617         entries = [
2618             unescapeHTML(mobj.group('url'))
2619             for mobj in re.finditer(r'''(?x)
2620             (?:
2621                 <iframe[^>]+?src=|
2622                 data-video-url=|
2623                 <embed[^>]+?src=|
2624                 embedSWF\(?:\s*|
2625                 <object[^>]+data=|
2626                 new\s+SWFObject\(
2627             )
2628             (["\'])
2629                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2630                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2631             \1''', webpage)]
2632
2633         # lazyYT YouTube embed
2634         entries.extend(list(map(
2635             unescapeHTML,
2636             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2637
2638         # Wordpress "YouTube Video Importer" plugin
2639         matches = re.findall(r'''(?x)<div[^>]+
2640             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2641             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2642         entries.extend(m[-1] for m in matches)
2643
2644         return entries
2645
2646     @staticmethod
2647     def _extract_url(webpage):
2648         urls = YoutubeIE._extract_urls(webpage)
2649         return urls[0] if urls else None
2650
2651     @classmethod
2652     def extract_id(cls, url):
2653         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2654         if mobj is None:
2655             raise ExtractorError('Invalid URL: %s' % url)
2656         return mobj.group('id')
2657
2658     def _extract_chapters_from_json(self, data, duration):
2659         chapter_list = traverse_obj(
2660             data, (
2661                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2662                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2663             ), expected_type=list)
2664
2665         return self._extract_chapters(
2666             chapter_list,
2667             chapter_time=lambda chapter: float_or_none(
2668                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2669             chapter_title=lambda chapter: traverse_obj(
2670                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2671             duration=duration)
2672
2673     def _extract_chapters_from_engagement_panel(self, data, duration):
2674         content_list = traverse_obj(
2675             data,
2676             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2677             expected_type=list, default=[])
2678         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2679         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2680
2681         return next((
2682             filter(None, (
2683                 self._extract_chapters(
2684                     traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2685                     chapter_time, chapter_title, duration)
2686                 for contents in content_list
2687             ))), [])
2688
2689     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration):
2690         chapters = []
2691         last_chapter = {'start_time': 0}
2692         for idx, chapter in enumerate(chapter_list or []):
2693             title = chapter_title(chapter)
2694             start_time = chapter_time(chapter)
2695             if start_time is None:
2696                 continue
2697             last_chapter['end_time'] = start_time
2698             if start_time < last_chapter['start_time']:
2699                 if idx == 1:
2700                     chapters.pop()
2701                     self.report_warning('Invalid start time for chapter "%s"' % last_chapter['title'])
2702                 else:
2703                     self.report_warning(f'Invalid start time for chapter "{title}"')
2704                     continue
2705             last_chapter = {'start_time': start_time, 'title': title}
2706             chapters.append(last_chapter)
2707         last_chapter['end_time'] = duration
2708         return chapters
2709
2710     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
2711         return self._parse_json(self._search_regex(
2712             (fr'{regex}\s*{self._YT_INITIAL_BOUNDARY_RE}',
2713              regex), webpage, name, default='{}'), video_id, fatal=False)
2714
2715     def _extract_comment(self, comment_renderer, parent=None):
2716         comment_id = comment_renderer.get('commentId')
2717         if not comment_id:
2718             return
2719
2720         text = self._get_text(comment_renderer, 'contentText')
2721
2722         # note: timestamp is an estimate calculated from the current time and time_text
2723         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2724         author = self._get_text(comment_renderer, 'authorText')
2725         author_id = try_get(comment_renderer,
2726                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2727
2728         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2729                                                        lambda x: x['likeCount']), compat_str)) or 0
2730         author_thumbnail = try_get(comment_renderer,
2731                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2732
2733         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2734         is_favorited = 'creatorHeart' in (try_get(
2735             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2736         return {
2737             'id': comment_id,
2738             'text': text,
2739             'timestamp': timestamp,
2740             'time_text': time_text,
2741             'like_count': votes,
2742             'is_favorited': is_favorited,
2743             'author': author,
2744             'author_id': author_id,
2745             'author_thumbnail': author_thumbnail,
2746             'author_is_uploader': author_is_uploader,
2747             'parent': parent or 'root'
2748         }
2749
2750     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2751
2752         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2753
2754         def extract_header(contents):
2755             _continuation = None
2756             for content in contents:
2757                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2758                 expected_comment_count = self._get_count(
2759                     comments_header_renderer, 'countText', 'commentsCount')
2760
2761                 if expected_comment_count:
2762                     tracker['est_total'] = expected_comment_count
2763                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2764                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2765
2766                 sort_menu_item = try_get(
2767                     comments_header_renderer,
2768                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2769                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2770
2771                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2772                 if not _continuation:
2773                     continue
2774
2775                 sort_text = str_or_none(sort_menu_item.get('title'))
2776                 if not sort_text:
2777                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2778                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2779                 break
2780             return _continuation
2781
2782         def extract_thread(contents):
2783             if not parent:
2784                 tracker['current_page_thread'] = 0
2785             for content in contents:
2786                 if not parent and tracker['total_parent_comments'] >= max_parents:
2787                     yield
2788                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2789                 comment_renderer = get_first(
2790                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2791                     expected_type=dict, default={})
2792
2793                 comment = self._extract_comment(comment_renderer, parent)
2794                 if not comment:
2795                     continue
2796
2797                 tracker['running_total'] += 1
2798                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2799                 yield comment
2800
2801                 # Attempt to get the replies
2802                 comment_replies_renderer = try_get(
2803                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2804
2805                 if comment_replies_renderer:
2806                     tracker['current_page_thread'] += 1
2807                     comment_entries_iter = self._comment_entries(
2808                         comment_replies_renderer, ytcfg, video_id,
2809                         parent=comment.get('id'), tracker=tracker)
2810                     yield from itertools.islice(comment_entries_iter, min(
2811                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
2812
2813         # Keeps track of counts across recursive calls
2814         if not tracker:
2815             tracker = dict(
2816                 running_total=0,
2817                 est_total=0,
2818                 current_page_thread=0,
2819                 total_parent_comments=0,
2820                 total_reply_comments=0)
2821
2822         # TODO: Deprecated
2823         # YouTube comments have a max depth of 2
2824         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2825         if max_depth:
2826             self._downloader.deprecation_warning(
2827                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2828         if max_depth == 1 and parent:
2829             return
2830
2831         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2832             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2833
2834         continuation = self._extract_continuation(root_continuation_data)
2835         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2836         if message and not parent:
2837             self.report_warning(message, video_id=video_id)
2838
2839         response = None
2840         is_first_continuation = parent is None
2841
2842         for page_num in itertools.count(0):
2843             if not continuation:
2844                 break
2845             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2846             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2847             if page_num == 0:
2848                 if is_first_continuation:
2849                     note_prefix = 'Downloading comment section API JSON'
2850                 else:
2851                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2852                         tracker['current_page_thread'], comment_prog_str)
2853             else:
2854                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2855                     '       ' if parent else '', ' replies' if parent else '',
2856                     page_num, comment_prog_str)
2857
2858             response = self._extract_response(
2859                 item_id=None, query=continuation,
2860                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2861                 check_get_keys='onResponseReceivedEndpoints')
2862
2863             continuation_contents = traverse_obj(
2864                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2865
2866             continuation = None
2867             for continuation_section in continuation_contents:
2868                 continuation_items = traverse_obj(
2869                     continuation_section,
2870                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2871                     get_all=False, expected_type=list) or []
2872                 if is_first_continuation:
2873                     continuation = extract_header(continuation_items)
2874                     is_first_continuation = False
2875                     if continuation:
2876                         break
2877                     continue
2878
2879                 for entry in extract_thread(continuation_items):
2880                     if not entry:
2881                         return
2882                     yield entry
2883                 continuation = self._extract_continuation({'contents': continuation_items})
2884                 if continuation:
2885                     break
2886
2887     def _get_comments(self, ytcfg, video_id, contents, webpage):
2888         """Entry for comment extraction"""
2889         def _real_comment_extract(contents):
2890             renderer = next((
2891                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2892                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2893             yield from self._comment_entries(renderer, ytcfg, video_id)
2894
2895         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2896         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2897
2898     @staticmethod
2899     def _get_checkok_params():
2900         return {'contentCheckOk': True, 'racyCheckOk': True}
2901
2902     @classmethod
2903     def _generate_player_context(cls, sts=None):
2904         context = {
2905             'html5Preference': 'HTML5_PREF_WANTS',
2906         }
2907         if sts is not None:
2908             context['signatureTimestamp'] = sts
2909         return {
2910             'playbackContext': {
2911                 'contentPlaybackContext': context
2912             },
2913             **cls._get_checkok_params()
2914         }
2915
2916     @staticmethod
2917     def _is_agegated(player_response):
2918         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2919             return True
2920
2921         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2922         AGE_GATE_REASONS = (
2923             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2924             'age_verification_required', 'age_check_required',  # status
2925         )
2926         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
2927
2928     @staticmethod
2929     def _is_unplayable(player_response):
2930         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
2931
2932     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
2933
2934         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
2935         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
2936         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
2937         headers = self.generate_api_headers(
2938             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
2939
2940         yt_query = {'videoId': video_id}
2941         yt_query.update(self._generate_player_context(sts))
2942         return self._extract_response(
2943             item_id=video_id, ep='player', query=yt_query,
2944             ytcfg=player_ytcfg, headers=headers, fatal=True,
2945             default_client=client,
2946             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
2947         ) or None
2948
2949     def _get_requested_clients(self, url, smuggled_data):
2950         requested_clients = []
2951         default = ['android', 'web']
2952         allowed_clients = sorted(
2953             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
2954             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
2955         for client in self._configuration_arg('player_client'):
2956             if client in allowed_clients:
2957                 requested_clients.append(client)
2958             elif client == 'default':
2959                 requested_clients.extend(default)
2960             elif client == 'all':
2961                 requested_clients.extend(allowed_clients)
2962             else:
2963                 self.report_warning(f'Skipping unsupported client {client}')
2964         if not requested_clients:
2965             requested_clients = default
2966
2967         if smuggled_data.get('is_music_url') or self.is_music_url(url):
2968             requested_clients.extend(
2969                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
2970
2971         return orderedSet(requested_clients)
2972
2973     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
2974         initial_pr = None
2975         if webpage:
2976             initial_pr = self._extract_yt_initial_variable(
2977                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
2978                 video_id, 'initial player response')
2979
2980         all_clients = set(clients)
2981         clients = clients[::-1]
2982         prs = []
2983
2984         def append_client(*client_names):
2985             """ Append the first client name that exists but not already used """
2986             for client_name in client_names:
2987                 actual_client = _split_innertube_client(client_name)[0]
2988                 if actual_client in INNERTUBE_CLIENTS:
2989                     if actual_client not in all_clients:
2990                         clients.append(client_name)
2991                         all_clients.add(actual_client)
2992                         return
2993
2994         # Android player_response does not have microFormats which are needed for
2995         # extraction of some data. So we return the initial_pr with formats
2996         # stripped out even if not requested by the user
2997         # See: https://github.com/yt-dlp/yt-dlp/issues/501
2998         if initial_pr:
2999             pr = dict(initial_pr)
3000             pr['streamingData'] = None
3001             prs.append(pr)
3002
3003         last_error = None
3004         tried_iframe_fallback = False
3005         player_url = None
3006         while clients:
3007             client, base_client, variant = _split_innertube_client(clients.pop())
3008             player_ytcfg = master_ytcfg if client == 'web' else {}
3009             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3010                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3011
3012             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3013             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3014             if 'js' in self._configuration_arg('player_skip'):
3015                 require_js_player = False
3016                 player_url = None
3017
3018             if not player_url and not tried_iframe_fallback and require_js_player:
3019                 player_url = self._download_player_url(video_id)
3020                 tried_iframe_fallback = True
3021
3022             try:
3023                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3024                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
3025             except ExtractorError as e:
3026                 if last_error:
3027                     self.report_warning(last_error)
3028                 last_error = e
3029                 continue
3030
3031             if pr:
3032                 prs.append(pr)
3033
3034             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3035             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3036                 append_client(f'{base_client}_creator')
3037             elif self._is_agegated(pr):
3038                 if variant == 'tv_embedded':
3039                     append_client(f'{base_client}_embedded')
3040                 elif not variant:
3041                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3042
3043         if last_error:
3044             if not len(prs):
3045                 raise last_error
3046             self.report_warning(last_error)
3047         return prs, player_url
3048
3049     def _extract_formats(self, streaming_data, video_id, player_url, is_live, duration):
3050         itags, stream_ids = {}, []
3051         itag_qualities, res_qualities = {}, {}
3052         q = qualities([
3053             # Normally tiny is the smallest video-only formats. But
3054             # audio-only formats with unknown quality may get tagged as tiny
3055             'tiny',
3056             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3057             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3058         ])
3059         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
3060
3061         for fmt in streaming_formats:
3062             if fmt.get('targetDurationSec'):
3063                 continue
3064
3065             itag = str_or_none(fmt.get('itag'))
3066             audio_track = fmt.get('audioTrack') or {}
3067             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
3068             if stream_id in stream_ids:
3069                 continue
3070
3071             quality = fmt.get('quality')
3072             height = int_or_none(fmt.get('height'))
3073             if quality == 'tiny' or not quality:
3074                 quality = fmt.get('audioQuality', '').lower() or quality
3075             # The 3gp format (17) in android client has a quality of "small",
3076             # but is actually worse than other formats
3077             if itag == '17':
3078                 quality = 'tiny'
3079             if quality:
3080                 if itag:
3081                     itag_qualities[itag] = quality
3082                 if height:
3083                     res_qualities[height] = quality
3084             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3085             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3086             # number of fragment that would subsequently requested with (`&sq=N`)
3087             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3088                 continue
3089
3090             fmt_url = fmt.get('url')
3091             if not fmt_url:
3092                 sc = compat_parse_qs(fmt.get('signatureCipher'))
3093                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3094                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3095                 if not (sc and fmt_url and encrypted_sig):
3096                     continue
3097                 if not player_url:
3098                     continue
3099                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
3100                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
3101                 fmt_url += '&' + sp + '=' + signature
3102
3103             query = parse_qs(fmt_url)
3104             throttled = False
3105             if query.get('n'):
3106                 try:
3107                     fmt_url = update_url_query(fmt_url, {
3108                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
3109                 except ExtractorError as e:
3110                     self.report_warning(
3111                         f'nsig extraction failed: You may experience throttling for some formats\n'
3112                         f'n = {query["n"][0]} ; player = {player_url}\n{e}', only_once=True)
3113                     throttled = True
3114
3115             if itag:
3116                 itags[itag] = 'https'
3117                 stream_ids.append(stream_id)
3118
3119             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3120             language_preference = (
3121                 10 if audio_track.get('audioIsDefault') and 10
3122                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3123                 else -1)
3124             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3125             # Eg: 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3126             # Make sure to avoid false positives with small duration differences.
3127             # Eg: __2ABJjxzNo, ySuUZEjARPY
3128             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3129             if is_damaged:
3130                 self.report_warning(f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3131             dct = {
3132                 'asr': int_or_none(fmt.get('audioSampleRate')),
3133                 'filesize': int_or_none(fmt.get('contentLength')),
3134                 'format_id': itag,
3135                 'format_note': join_nonempty(
3136                     '%s%s' % (audio_track.get('displayName') or '',
3137                               ' (default)' if language_preference > 0 else ''),
3138                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3139                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3140                 'source_preference': -10 if throttled else -1,
3141                 'fps': int_or_none(fmt.get('fps')) or None,
3142                 'height': height,
3143                 'quality': q(quality),
3144                 'has_drm': bool(fmt.get('drmFamilies')),
3145                 'tbr': tbr,
3146                 'url': fmt_url,
3147                 'width': int_or_none(fmt.get('width')),
3148                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3149                                           'desc' if language_preference < -1 else ''),
3150                 'language_preference': language_preference,
3151                 # Strictly de-prioritize damaged and 3gp formats
3152                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3153             }
3154             mime_mobj = re.match(
3155                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3156             if mime_mobj:
3157                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3158                 dct.update(parse_codecs(mime_mobj.group(2)))
3159             no_audio = dct.get('acodec') == 'none'
3160             no_video = dct.get('vcodec') == 'none'
3161             if no_audio:
3162                 dct['vbr'] = tbr
3163             if no_video:
3164                 dct['abr'] = tbr
3165             if no_audio or no_video:
3166                 dct['downloader_options'] = {
3167                     # Youtube throttles chunks >~10M
3168                     'http_chunk_size': 10485760,
3169                 }
3170                 if dct.get('ext'):
3171                     dct['container'] = dct['ext'] + '_dash'
3172             yield dct
3173
3174         live_from_start = is_live and self.get_param('live_from_start')
3175         skip_manifests = self._configuration_arg('skip')
3176         if not self.get_param('youtube_include_hls_manifest', True):
3177             skip_manifests.append('hls')
3178         get_dash = 'dash' not in skip_manifests and (
3179             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3180         get_hls = not live_from_start and 'hls' not in skip_manifests
3181
3182         def process_manifest_format(f, proto, itag):
3183             if itag in itags:
3184                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3185                     return False
3186                 itag = f'{itag}-{proto}'
3187             if itag:
3188                 f['format_id'] = itag
3189                 itags[itag] = proto
3190
3191             f['quality'] = next((
3192                 q(qdict[val])
3193                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3194                 if val in qdict), -1)
3195             return True
3196
3197         for sd in streaming_data:
3198             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3199             if hls_manifest_url:
3200                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3201                     if process_manifest_format(f, 'hls', self._search_regex(
3202                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3203                         yield f
3204
3205             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3206             if dash_manifest_url:
3207                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3208                     if process_manifest_format(f, 'dash', f['format_id']):
3209                         f['filesize'] = int_or_none(self._search_regex(
3210                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3211                         if live_from_start:
3212                             f['is_from_start'] = True
3213
3214                         yield f
3215
3216     def _extract_storyboard(self, player_responses, duration):
3217         spec = get_first(
3218             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3219         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3220         if not base_url:
3221             return
3222         L = len(spec) - 1
3223         for i, args in enumerate(spec):
3224             args = args.split('#')
3225             counts = list(map(int_or_none, args[:5]))
3226             if len(args) != 8 or not all(counts):
3227                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3228                 continue
3229             width, height, frame_count, cols, rows = counts
3230             N, sigh = args[6:]
3231
3232             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3233             fragment_count = frame_count / (cols * rows)
3234             fragment_duration = duration / fragment_count
3235             yield {
3236                 'format_id': f'sb{i}',
3237                 'format_note': 'storyboard',
3238                 'ext': 'mhtml',
3239                 'protocol': 'mhtml',
3240                 'acodec': 'none',
3241                 'vcodec': 'none',
3242                 'url': url,
3243                 'width': width,
3244                 'height': height,
3245                 'fragments': [{
3246                     'url': url.replace('$M', str(j)),
3247                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3248                 } for j in range(math.ceil(fragment_count))],
3249             }
3250
3251     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3252         webpage = None
3253         if 'webpage' not in self._configuration_arg('player_skip'):
3254             webpage = self._download_webpage(
3255                 webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
3256
3257         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3258
3259         player_responses, player_url = self._extract_player_responses(
3260             self._get_requested_clients(url, smuggled_data),
3261             video_id, webpage, master_ytcfg)
3262
3263         return webpage, master_ytcfg, player_responses, player_url
3264
3265     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3266         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3267         is_live = get_first(video_details, 'isLive')
3268         if is_live is None:
3269             is_live = get_first(live_broadcast_details, 'isLiveNow')
3270
3271         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3272         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live, duration))
3273
3274         return live_broadcast_details, is_live, streaming_data, formats
3275
3276     def _real_extract(self, url):
3277         url, smuggled_data = unsmuggle_url(url, {})
3278         video_id = self._match_id(url)
3279
3280         base_url = self.http_scheme() + '//www.youtube.com/'
3281         webpage_url = base_url + 'watch?v=' + video_id
3282
3283         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3284
3285         playability_statuses = traverse_obj(
3286             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3287
3288         trailer_video_id = get_first(
3289             playability_statuses,
3290             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3291             expected_type=str)
3292         if trailer_video_id:
3293             return self.url_result(
3294                 trailer_video_id, self.ie_key(), trailer_video_id)
3295
3296         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3297                        if webpage else (lambda x: None))
3298
3299         video_details = traverse_obj(
3300             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3301         microformats = traverse_obj(
3302             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3303             expected_type=dict, default=[])
3304         video_title = (
3305             get_first(video_details, 'title')
3306             or self._get_text(microformats, (..., 'title'))
3307             or search_meta(['og:title', 'twitter:title', 'title']))
3308         video_description = get_first(video_details, 'shortDescription')
3309
3310         multifeed_metadata_list = get_first(
3311             player_responses,
3312             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3313             expected_type=str)
3314         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3315             if self.get_param('noplaylist'):
3316                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3317             else:
3318                 entries = []
3319                 feed_ids = []
3320                 for feed in multifeed_metadata_list.split(','):
3321                     # Unquote should take place before split on comma (,) since textual
3322                     # fields may contain comma as well (see
3323                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3324                     feed_data = compat_parse_qs(
3325                         compat_urllib_parse_unquote_plus(feed))
3326
3327                     def feed_entry(name):
3328                         return try_get(
3329                             feed_data, lambda x: x[name][0], compat_str)
3330
3331                     feed_id = feed_entry('id')
3332                     if not feed_id:
3333                         continue
3334                     feed_title = feed_entry('title')
3335                     title = video_title
3336                     if feed_title:
3337                         title += ' (%s)' % feed_title
3338                     entries.append({
3339                         '_type': 'url_transparent',
3340                         'ie_key': 'Youtube',
3341                         'url': smuggle_url(
3342                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3343                             {'force_singlefeed': True}),
3344                         'title': title,
3345                     })
3346                     feed_ids.append(feed_id)
3347                 self.to_screen(
3348                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3349                     % (', '.join(feed_ids), video_id))
3350                 return self.playlist_result(
3351                     entries, video_id, video_title, video_description)
3352
3353         duration = int_or_none(
3354             get_first(video_details, 'lengthSeconds')
3355             or get_first(microformats, 'lengthSeconds')
3356             or parse_duration(search_meta('duration'))) or None
3357
3358         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(
3359             video_id, microformats, video_details, player_responses, player_url, duration)
3360
3361         if not formats:
3362             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3363                 self.report_drm(video_id)
3364             pemr = get_first(
3365                 playability_statuses,
3366                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3367             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3368             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3369             if subreason:
3370                 if subreason == 'The uploader has not made this video available in your country.':
3371                     countries = get_first(microformats, 'availableCountries')
3372                     if not countries:
3373                         regions_allowed = search_meta('regionsAllowed')
3374                         countries = regions_allowed.split(',') if regions_allowed else None
3375                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3376                 reason += f'. {subreason}'
3377             if reason:
3378                 self.raise_no_formats(reason, expected=True)
3379
3380         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3381         if not keywords and webpage:
3382             keywords = [
3383                 unescapeHTML(m.group('content'))
3384                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3385         for keyword in keywords:
3386             if keyword.startswith('yt:stretch='):
3387                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3388                 if mobj:
3389                     # NB: float is intentional for forcing float division
3390                     w, h = (float(v) for v in mobj.groups())
3391                     if w > 0 and h > 0:
3392                         ratio = w / h
3393                         for f in formats:
3394                             if f.get('vcodec') != 'none':
3395                                 f['stretched_ratio'] = ratio
3396                         break
3397         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3398         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3399         if thumbnail_url:
3400             thumbnails.append({
3401                 'url': thumbnail_url,
3402             })
3403         original_thumbnails = thumbnails.copy()
3404
3405         # The best resolution thumbnails sometimes does not appear in the webpage
3406         # See: https://github.com/ytdl-org/youtube-dl/issues/29049, https://github.com/yt-dlp/yt-dlp/issues/340
3407         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3408         thumbnail_names = [
3409             'maxresdefault', 'hq720', 'sddefault', 'sd1', 'sd2', 'sd3',
3410             'hqdefault', 'hq1', 'hq2', 'hq3', '0',
3411             'mqdefault', 'mq1', 'mq2', 'mq3',
3412             'default', '1', '2', '3'
3413         ]
3414         n_thumbnail_names = len(thumbnail_names)
3415         thumbnails.extend({
3416             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3417                 video_id=video_id, name=name, ext=ext,
3418                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3419         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3420         for thumb in thumbnails:
3421             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3422             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3423         self._remove_duplicate_formats(thumbnails)
3424         self._downloader._sort_thumbnails(original_thumbnails)
3425
3426         category = get_first(microformats, 'category') or search_meta('genre')
3427         channel_id = str_or_none(
3428             get_first(video_details, 'channelId')
3429             or get_first(microformats, 'externalChannelId')
3430             or search_meta('channelId'))
3431         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3432
3433         live_content = get_first(video_details, 'isLiveContent')
3434         is_upcoming = get_first(video_details, 'isUpcoming')
3435         if is_live is None:
3436             if is_upcoming or live_content is False:
3437                 is_live = False
3438         if is_upcoming is None and (live_content or is_live):
3439             is_upcoming = False
3440         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3441         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3442         if not duration and live_end_time and live_start_time:
3443             duration = live_end_time - live_start_time
3444
3445         if is_live and self.get_param('live_from_start'):
3446             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3447
3448         formats.extend(self._extract_storyboard(player_responses, duration))
3449
3450         # Source is given priority since formats that throttle are given lower source_preference
3451         # When throttling issue is fully fixed, remove this
3452         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3453
3454         info = {
3455             'id': video_id,
3456             'title': video_title,
3457             'formats': formats,
3458             'thumbnails': thumbnails,
3459             # The best thumbnail that we are sure exists. Prevents unnecessary
3460             # URL checking if user don't care about getting the best possible thumbnail
3461             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3462             'description': video_description,
3463             'uploader': get_first(video_details, 'author'),
3464             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3465             'uploader_url': owner_profile_url,
3466             'channel_id': channel_id,
3467             'channel_url': format_field(channel_id, template='https://www.youtube.com/channel/%s'),
3468             'duration': duration,
3469             'view_count': int_or_none(
3470                 get_first((video_details, microformats), (..., 'viewCount'))
3471                 or search_meta('interactionCount')),
3472             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3473             'age_limit': 18 if (
3474                 get_first(microformats, 'isFamilySafe') is False
3475                 or search_meta('isFamilyFriendly') == 'false'
3476                 or search_meta('og:restrictions:age') == '18+') else 0,
3477             'webpage_url': webpage_url,
3478             'categories': [category] if category else None,
3479             'tags': keywords,
3480             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3481             'is_live': is_live,
3482             'was_live': (False if is_live or is_upcoming or live_content is False
3483                          else None if is_live is None or is_upcoming is None
3484                          else live_content),
3485             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3486             'release_timestamp': live_start_time,
3487         }
3488
3489         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3490         if pctr:
3491             def get_lang_code(track):
3492                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3493                         or track.get('languageCode'))
3494
3495             # Converted into dicts to remove duplicates
3496             captions = {
3497                 get_lang_code(sub): sub
3498                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3499             translation_languages = {
3500                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3501                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3502
3503             def process_language(container, base_url, lang_code, sub_name, query):
3504                 lang_subs = container.setdefault(lang_code, [])
3505                 for fmt in self._SUBTITLE_FORMATS:
3506                     query.update({
3507                         'fmt': fmt,
3508                     })
3509                     lang_subs.append({
3510                         'ext': fmt,
3511                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
3512                         'name': sub_name,
3513                     })
3514
3515             subtitles, automatic_captions = {}, {}
3516             for lang_code, caption_track in captions.items():
3517                 base_url = caption_track.get('baseUrl')
3518                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
3519                 if not base_url:
3520                     continue
3521                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3522                 if caption_track.get('kind') != 'asr':
3523                     if not lang_code:
3524                         continue
3525                     process_language(
3526                         subtitles, base_url, lang_code, lang_name, {})
3527                     if not caption_track.get('isTranslatable'):
3528                         continue
3529                 for trans_code, trans_name in translation_languages.items():
3530                     if not trans_code:
3531                         continue
3532                     orig_trans_code = trans_code
3533                     if caption_track.get('kind') != 'asr':
3534                         if 'translated_subs' in self._configuration_arg('skip'):
3535                             continue
3536                         trans_code += f'-{lang_code}'
3537                         trans_name += format_field(lang_name, template=' from %s')
3538                     # Add an "-orig" label to the original language so that it can be distinguished.
3539                     # The subs are returned without "-orig" as well for compatibility
3540                     if lang_code == f'a-{orig_trans_code}':
3541                         process_language(
3542                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
3543                     # Setting tlang=lang returns damaged subtitles.
3544                     process_language(automatic_captions, base_url, trans_code, trans_name,
3545                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
3546             info['automatic_captions'] = automatic_captions
3547             info['subtitles'] = subtitles
3548
3549         parsed_url = compat_urllib_parse_urlparse(url)
3550         for component in [parsed_url.fragment, parsed_url.query]:
3551             query = compat_parse_qs(component)
3552             for k, v in query.items():
3553                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3554                     d_k += '_time'
3555                     if d_k not in info and k in s_ks:
3556                         info[d_k] = parse_duration(query[k][0])
3557
3558         # Youtube Music Auto-generated description
3559         if video_description:
3560             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
3561             if mobj:
3562                 release_year = mobj.group('release_year')
3563                 release_date = mobj.group('release_date')
3564                 if release_date:
3565                     release_date = release_date.replace('-', '')
3566                     if not release_year:
3567                         release_year = release_date[:4]
3568                 info.update({
3569                     'album': mobj.group('album'.strip()),
3570                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3571                     'track': mobj.group('track').strip(),
3572                     'release_date': release_date,
3573                     'release_year': int_or_none(release_year),
3574                 })
3575
3576         initial_data = None
3577         if webpage:
3578             initial_data = self._extract_yt_initial_variable(
3579                 webpage, self._YT_INITIAL_DATA_RE, video_id,
3580                 'yt initial data')
3581         if not initial_data:
3582             query = {'videoId': video_id}
3583             query.update(self._get_checkok_params())
3584             initial_data = self._extract_response(
3585                 item_id=video_id, ep='next', fatal=False,
3586                 ytcfg=master_ytcfg, query=query,
3587                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3588                 note='Downloading initial data API JSON')
3589
3590         try:
3591             # This will error if there is no livechat
3592             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3593             info.setdefault('subtitles', {})['live_chat'] = [{
3594                 'url': 'https://www.youtube.com/watch?v=%s' % video_id,  # url is needed to set cookies
3595                 'video_id': video_id,
3596                 'ext': 'json',
3597                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3598             }]
3599         except (KeyError, IndexError, TypeError):
3600             pass
3601
3602         if initial_data:
3603             info['chapters'] = (
3604                 self._extract_chapters_from_json(initial_data, duration)
3605                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3606                 or None)
3607
3608         contents = traverse_obj(
3609             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
3610             expected_type=list, default=[])
3611
3612         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
3613         if vpir:
3614             stl = vpir.get('superTitleLink')
3615             if stl:
3616                 stl = self._get_text(stl)
3617                 if try_get(
3618                         vpir,
3619                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3620                     info['location'] = stl
3621                 else:
3622                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•\s*E(\d+)', stl)
3623                     if mobj:
3624                         info.update({
3625                             'series': mobj.group(1),
3626                             'season_number': int(mobj.group(2)),
3627                             'episode_number': int(mobj.group(3)),
3628                         })
3629             for tlb in (try_get(
3630                     vpir,
3631                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3632                     list) or []):
3633                 tbr = tlb.get('toggleButtonRenderer') or {}
3634                 for getter, regex in [(
3635                         lambda x: x['defaultText']['accessibility']['accessibilityData'],
3636                         r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3637                             lambda x: x['accessibility'],
3638                             lambda x: x['accessibilityData']['accessibilityData'],
3639                         ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3640                     label = (try_get(tbr, getter, dict) or {}).get('label')
3641                     if label:
3642                         mobj = re.match(regex, label)
3643                         if mobj:
3644                             info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3645                             break
3646             sbr_tooltip = try_get(
3647                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3648             if sbr_tooltip:
3649                 like_count, dislike_count = sbr_tooltip.split(' / ')
3650                 info.update({
3651                     'like_count': str_to_int(like_count),
3652                     'dislike_count': str_to_int(dislike_count),
3653                 })
3654         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
3655         if vsir:
3656             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3657             info.update({
3658                 'channel': self._get_text(vor, 'title'),
3659                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3660
3661             rows = try_get(
3662                 vsir,
3663                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3664                 list) or []
3665             multiple_songs = False
3666             for row in rows:
3667                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3668                     multiple_songs = True
3669                     break
3670             for row in rows:
3671                 mrr = row.get('metadataRowRenderer') or {}
3672                 mrr_title = mrr.get('title')
3673                 if not mrr_title:
3674                     continue
3675                 mrr_title = self._get_text(mrr, 'title')
3676                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
3677                 if mrr_title == 'License':
3678                     info['license'] = mrr_contents_text
3679                 elif not multiple_songs:
3680                     if mrr_title == 'Album':
3681                         info['album'] = mrr_contents_text
3682                     elif mrr_title == 'Artist':
3683                         info['artist'] = mrr_contents_text
3684                     elif mrr_title == 'Song':
3685                         info['track'] = mrr_contents_text
3686
3687         fallbacks = {
3688             'channel': 'uploader',
3689             'channel_id': 'uploader_id',
3690             'channel_url': 'uploader_url',
3691         }
3692
3693         # The upload date for scheduled, live and past live streams / premieres in microformats
3694         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
3695         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
3696         upload_date = (
3697             unified_strdate(get_first(microformats, 'uploadDate'))
3698             or unified_strdate(search_meta('uploadDate')))
3699         if not upload_date or (not info.get('is_live') and not info.get('was_live') and info.get('live_status') != 'is_upcoming'):
3700             upload_date = strftime_or_none(self._extract_time_text(vpir, 'dateText')[0], '%Y%m%d')
3701         info['upload_date'] = upload_date
3702
3703         for to, frm in fallbacks.items():
3704             if not info.get(to):
3705                 info[to] = info.get(frm)
3706
3707         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3708             v = info.get(s_k)
3709             if v:
3710                 info[d_k] = v
3711
3712         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3713         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3714         is_membersonly = None
3715         is_premium = None
3716         if initial_data and is_private is not None:
3717             is_membersonly = False
3718             is_premium = False
3719             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3720             badge_labels = set()
3721             for content in contents:
3722                 if not isinstance(content, dict):
3723                     continue
3724                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3725             for badge_label in badge_labels:
3726                 if badge_label.lower() == 'members only':
3727                     is_membersonly = True
3728                 elif badge_label.lower() == 'premium':
3729                     is_premium = True
3730                 elif badge_label.lower() == 'unlisted':
3731                     is_unlisted = True
3732
3733         info['availability'] = self._availability(
3734             is_private=is_private,
3735             needs_premium=is_premium,
3736             needs_subscription=is_membersonly,
3737             needs_auth=info['age_limit'] >= 18,
3738             is_unlisted=None if is_private is None else is_unlisted)
3739
3740         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3741
3742         self.mark_watched(video_id, player_responses)
3743
3744         return info
3745
3746
3747 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3748
3749     @staticmethod
3750     def passthrough_smuggled_data(func):
3751         def _smuggle(entries, smuggled_data):
3752             for entry in entries:
3753                 # TODO: Convert URL to music.youtube instead.
3754                 # Do we need to passthrough any other smuggled_data?
3755                 entry['url'] = smuggle_url(entry['url'], smuggled_data)
3756                 yield entry
3757
3758         @functools.wraps(func)
3759         def wrapper(self, url):
3760             url, smuggled_data = unsmuggle_url(url, {})
3761             if self.is_music_url(url):
3762                 smuggled_data['is_music_url'] = True
3763             info_dict = func(self, url, smuggled_data)
3764             if smuggled_data and info_dict.get('entries'):
3765                 info_dict['entries'] = _smuggle(info_dict['entries'], smuggled_data)
3766             return info_dict
3767         return wrapper
3768
3769     def _extract_channel_id(self, webpage):
3770         channel_id = self._html_search_meta(
3771             'channelId', webpage, 'channel id', default=None)
3772         if channel_id:
3773             return channel_id
3774         channel_url = self._html_search_meta(
3775             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3776              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3777              'twitter:app:url:googleplay'), webpage, 'channel url')
3778         return self._search_regex(
3779             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3780             channel_url, 'channel id')
3781
3782     @staticmethod
3783     def _extract_basic_item_renderer(item):
3784         # Modified from _extract_grid_item_renderer
3785         known_basic_renderers = (
3786             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
3787         )
3788         for key, renderer in item.items():
3789             if not isinstance(renderer, dict):
3790                 continue
3791             elif key in known_basic_renderers:
3792                 return renderer
3793             elif key.startswith('grid') and key.endswith('Renderer'):
3794                 return renderer
3795
3796     def _grid_entries(self, grid_renderer):
3797         for item in grid_renderer['items']:
3798             if not isinstance(item, dict):
3799                 continue
3800             renderer = self._extract_basic_item_renderer(item)
3801             if not isinstance(renderer, dict):
3802                 continue
3803             title = self._get_text(renderer, 'title')
3804
3805             # playlist
3806             playlist_id = renderer.get('playlistId')
3807             if playlist_id:
3808                 yield self.url_result(
3809                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3810                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3811                     video_title=title)
3812                 continue
3813             # video
3814             video_id = renderer.get('videoId')
3815             if video_id:
3816                 yield self._extract_video(renderer)
3817                 continue
3818             # channel
3819             channel_id = renderer.get('channelId')
3820             if channel_id:
3821                 yield self.url_result(
3822                     'https://www.youtube.com/channel/%s' % channel_id,
3823                     ie=YoutubeTabIE.ie_key(), video_title=title)
3824                 continue
3825             # generic endpoint URL support
3826             ep_url = urljoin('https://www.youtube.com/', try_get(
3827                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3828                 compat_str))
3829             if ep_url:
3830                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3831                     if ie.suitable(ep_url):
3832                         yield self.url_result(
3833                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3834                         break
3835
3836     def _music_reponsive_list_entry(self, renderer):
3837         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
3838         if video_id:
3839             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
3840                                    ie=YoutubeIE.ie_key(), video_id=video_id)
3841         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
3842         if playlist_id:
3843             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
3844             if video_id:
3845                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
3846                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3847             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
3848                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3849         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
3850         if browse_id:
3851             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
3852                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
3853
3854     def _shelf_entries_from_content(self, shelf_renderer):
3855         content = shelf_renderer.get('content')
3856         if not isinstance(content, dict):
3857             return
3858         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3859         if renderer:
3860             # TODO: add support for nested playlists so each shelf is processed
3861             # as separate playlist
3862             # TODO: this includes only first N items
3863             yield from self._grid_entries(renderer)
3864         renderer = content.get('horizontalListRenderer')
3865         if renderer:
3866             # TODO
3867             pass
3868
3869     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3870         ep = try_get(
3871             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3872             compat_str)
3873         shelf_url = urljoin('https://www.youtube.com', ep)
3874         if shelf_url:
3875             # Skipping links to another channels, note that checking for
3876             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3877             # will not work
3878             if skip_channels and '/channels?' in shelf_url:
3879                 return
3880             title = self._get_text(shelf_renderer, 'title')
3881             yield self.url_result(shelf_url, video_title=title)
3882         # Shelf may not contain shelf URL, fallback to extraction from content
3883         yield from self._shelf_entries_from_content(shelf_renderer)
3884
3885     def _playlist_entries(self, video_list_renderer):
3886         for content in video_list_renderer['contents']:
3887             if not isinstance(content, dict):
3888                 continue
3889             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3890             if not isinstance(renderer, dict):
3891                 continue
3892             video_id = renderer.get('videoId')
3893             if not video_id:
3894                 continue
3895             yield self._extract_video(renderer)
3896
3897     def _rich_entries(self, rich_grid_renderer):
3898         renderer = try_get(
3899             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3900         video_id = renderer.get('videoId')
3901         if not video_id:
3902             return
3903         yield self._extract_video(renderer)
3904
3905     def _video_entry(self, video_renderer):
3906         video_id = video_renderer.get('videoId')
3907         if video_id:
3908             return self._extract_video(video_renderer)
3909
3910     def _hashtag_tile_entry(self, hashtag_tile_renderer):
3911         url = urljoin('https://youtube.com', traverse_obj(
3912             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
3913         if url:
3914             return self.url_result(
3915                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
3916
3917     def _post_thread_entries(self, post_thread_renderer):
3918         post_renderer = try_get(
3919             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
3920         if not post_renderer:
3921             return
3922         # video attachment
3923         video_renderer = try_get(
3924             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
3925         video_id = video_renderer.get('videoId')
3926         if video_id:
3927             entry = self._extract_video(video_renderer)
3928             if entry:
3929                 yield entry
3930         # playlist attachment
3931         playlist_id = try_get(
3932             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
3933         if playlist_id:
3934             yield self.url_result(
3935                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
3936                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3937         # inline video links
3938         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
3939         for run in runs:
3940             if not isinstance(run, dict):
3941                 continue
3942             ep_url = try_get(
3943                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
3944             if not ep_url:
3945                 continue
3946             if not YoutubeIE.suitable(ep_url):
3947                 continue
3948             ep_video_id = YoutubeIE._match_id(ep_url)
3949             if video_id == ep_video_id:
3950                 continue
3951             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
3952
3953     def _post_thread_continuation_entries(self, post_thread_continuation):
3954         contents = post_thread_continuation.get('contents')
3955         if not isinstance(contents, list):
3956             return
3957         for content in contents:
3958             renderer = content.get('backstagePostThreadRenderer')
3959             if not isinstance(renderer, dict):
3960                 continue
3961             yield from self._post_thread_entries(renderer)
3962
3963     r''' # unused
3964     def _rich_grid_entries(self, contents):
3965         for content in contents:
3966             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
3967             if video_renderer:
3968                 entry = self._video_entry(video_renderer)
3969                 if entry:
3970                     yield entry
3971     '''
3972
3973     def _extract_entries(self, parent_renderer, continuation_list):
3974         # continuation_list is modified in-place with continuation_list = [continuation_token]
3975         continuation_list[:] = [None]
3976         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
3977         for content in contents:
3978             if not isinstance(content, dict):
3979                 continue
3980             is_renderer = traverse_obj(
3981                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
3982                 expected_type=dict)
3983             if not is_renderer:
3984                 renderer = content.get('richItemRenderer')
3985                 if renderer:
3986                     for entry in self._rich_entries(renderer):
3987                         yield entry
3988                     continuation_list[0] = self._extract_continuation(parent_renderer)
3989                 continue
3990             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
3991             for isr_content in isr_contents:
3992                 if not isinstance(isr_content, dict):
3993                     continue
3994
3995                 known_renderers = {
3996                     'playlistVideoListRenderer': self._playlist_entries,
3997                     'gridRenderer': self._grid_entries,
3998                     'reelShelfRenderer': self._grid_entries,
3999                     'shelfRenderer': self._shelf_entries,
4000                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4001                     'backstagePostThreadRenderer': self._post_thread_entries,
4002                     'videoRenderer': lambda x: [self._video_entry(x)],
4003                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4004                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4005                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4006                 }
4007                 for key, renderer in isr_content.items():
4008                     if key not in known_renderers:
4009                         continue
4010                     for entry in known_renderers[key](renderer):
4011                         if entry:
4012                             yield entry
4013                     continuation_list[0] = self._extract_continuation(renderer)
4014                     break
4015
4016             if not continuation_list[0]:
4017                 continuation_list[0] = self._extract_continuation(is_renderer)
4018
4019         if not continuation_list[0]:
4020             continuation_list[0] = self._extract_continuation(parent_renderer)
4021
4022     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4023         continuation_list = [None]
4024         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4025         tab_content = try_get(tab, lambda x: x['content'], dict)
4026         if not tab_content:
4027             return
4028         parent_renderer = (
4029             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4030             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4031         yield from extract_entries(parent_renderer)
4032         continuation = continuation_list[0]
4033
4034         for page_num in itertools.count(1):
4035             if not continuation:
4036                 break
4037             headers = self.generate_api_headers(
4038                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4039             response = self._extract_response(
4040                 item_id=f'{item_id} page {page_num}',
4041                 query=continuation, headers=headers, ytcfg=ytcfg,
4042                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4043
4044             if not response:
4045                 break
4046             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4047             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4048             visitor_data = self._extract_visitor_data(response) or visitor_data
4049
4050             known_continuation_renderers = {
4051                 'playlistVideoListContinuation': self._playlist_entries,
4052                 'gridContinuation': self._grid_entries,
4053                 'itemSectionContinuation': self._post_thread_continuation_entries,
4054                 'sectionListContinuation': extract_entries,  # for feeds
4055             }
4056             continuation_contents = try_get(
4057                 response, lambda x: x['continuationContents'], dict) or {}
4058             continuation_renderer = None
4059             for key, value in continuation_contents.items():
4060                 if key not in known_continuation_renderers:
4061                     continue
4062                 continuation_renderer = value
4063                 continuation_list = [None]
4064                 yield from known_continuation_renderers[key](continuation_renderer)
4065                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
4066                 break
4067             if continuation_renderer:
4068                 continue
4069
4070             known_renderers = {
4071                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4072                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4073                 'gridVideoRenderer': (self._grid_entries, 'items'),
4074                 'gridChannelRenderer': (self._grid_entries, 'items'),
4075                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4076                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4077                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4078                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
4079             }
4080             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4081             continuation_items = try_get(
4082                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
4083             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
4084             video_items_renderer = None
4085             for key, value in continuation_item.items():
4086                 if key not in known_renderers:
4087                     continue
4088                 video_items_renderer = {known_renderers[key][1]: continuation_items}
4089                 continuation_list = [None]
4090                 yield from known_renderers[key][0](video_items_renderer)
4091                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4092                 break
4093             if video_items_renderer:
4094                 continue
4095             break
4096
4097     @staticmethod
4098     def _extract_selected_tab(tabs, fatal=True):
4099         for tab in tabs:
4100             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
4101             if renderer.get('selected') is True:
4102                 return renderer
4103         else:
4104             if fatal:
4105                 raise ExtractorError('Unable to find selected tab')
4106
4107     def _extract_uploader(self, data):
4108         uploader = {}
4109         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
4110         owner = try_get(
4111             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
4112         if owner:
4113             owner_text = owner.get('text')
4114             uploader['uploader'] = self._search_regex(
4115                 r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text)
4116             uploader['uploader_id'] = try_get(
4117                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
4118             uploader['uploader_url'] = urljoin(
4119                 'https://www.youtube.com/',
4120                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
4121         return {k: v for k, v in uploader.items() if v is not None}
4122
4123     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4124         playlist_id = title = description = channel_url = channel_name = channel_id = None
4125         tags = []
4126
4127         selected_tab = self._extract_selected_tab(tabs)
4128         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4129         renderer = try_get(
4130             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
4131         if renderer:
4132             channel_name = renderer.get('title')
4133             channel_url = renderer.get('channelUrl')
4134             channel_id = renderer.get('externalId')
4135         else:
4136             renderer = try_get(
4137                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
4138
4139         if renderer:
4140             title = renderer.get('title')
4141             description = renderer.get('description', '')
4142             playlist_id = channel_id
4143             tags = renderer.get('keywords', '').split()
4144
4145         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4146         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4147         def _get_uncropped(url):
4148             return url_or_none((url or '').split('=')[0] + '=s0')
4149
4150         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
4151         if avatar_thumbnails:
4152             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4153             if uncropped_avatar:
4154                 avatar_thumbnails.append({
4155                     'url': uncropped_avatar,
4156                     'id': 'avatar_uncropped',
4157                     'preference': 1
4158                 })
4159
4160         channel_banners = self._extract_thumbnails(
4161             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
4162         for banner in channel_banners:
4163             banner['preference'] = -10
4164
4165         if channel_banners:
4166             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4167             if uncropped_banner:
4168                 channel_banners.append({
4169                     'url': uncropped_banner,
4170                     'id': 'banner_uncropped',
4171                     'preference': -5
4172                 })
4173
4174         primary_thumbnails = self._extract_thumbnails(
4175             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4176
4177         if playlist_id is None:
4178             playlist_id = item_id
4179
4180         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
4181         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
4182         if title is None:
4183             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
4184         title += format_field(selected_tab, 'title', ' - %s')
4185         title += format_field(selected_tab, 'expandedText', ' - %s')
4186
4187         metadata = {
4188             'playlist_id': playlist_id,
4189             'playlist_title': title,
4190             'playlist_description': description,
4191             'uploader': channel_name,
4192             'uploader_id': channel_id,
4193             'uploader_url': channel_url,
4194             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
4195             'tags': tags,
4196             'view_count': self._get_count(playlist_stats, 1),
4197             'availability': self._extract_availability(data),
4198             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4199             'playlist_count': self._get_count(playlist_stats, 0),
4200             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4201         }
4202         if not channel_id:
4203             metadata.update(self._extract_uploader(data))
4204         metadata.update({
4205             'channel': metadata['uploader'],
4206             'channel_id': metadata['uploader_id'],
4207             'channel_url': metadata['uploader_url']})
4208         return self.playlist_result(
4209             self._entries(
4210                 selected_tab, playlist_id, ytcfg,
4211                 self._extract_account_syncid(ytcfg, data),
4212                 self._extract_visitor_data(data, ytcfg)),
4213             **metadata)
4214
4215     def _extract_mix_playlist(self, playlist, playlist_id, data, ytcfg):
4216         first_id = last_id = response = None
4217         for page_num in itertools.count(1):
4218             videos = list(self._playlist_entries(playlist))
4219             if not videos:
4220                 return
4221             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4222             if start >= len(videos):
4223                 return
4224             for video in videos[start:]:
4225                 if video['id'] == first_id:
4226                     self.to_screen('First video %s found again; Assuming end of Mix' % first_id)
4227                     return
4228                 yield video
4229             first_id = first_id or videos[0]['id']
4230             last_id = videos[-1]['id']
4231             watch_endpoint = try_get(
4232                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4233             headers = self.generate_api_headers(
4234                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4235                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4236             query = {
4237                 'playlistId': playlist_id,
4238                 'videoId': watch_endpoint.get('videoId') or last_id,
4239                 'index': watch_endpoint.get('index') or len(videos),
4240                 'params': watch_endpoint.get('params') or 'OAE%3D'
4241             }
4242             response = self._extract_response(
4243                 item_id='%s page %d' % (playlist_id, page_num),
4244                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4245                 check_get_keys='contents'
4246             )
4247             playlist = try_get(
4248                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4249
4250     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4251         title = playlist.get('title') or try_get(
4252             data, lambda x: x['titleText']['simpleText'], compat_str)
4253         playlist_id = playlist.get('playlistId') or item_id
4254
4255         # Delegating everything except mix playlists to regular tab-based playlist URL
4256         playlist_url = urljoin(url, try_get(
4257             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4258             compat_str))
4259         if playlist_url and playlist_url != url:
4260             return self.url_result(
4261                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4262                 video_title=title)
4263
4264         return self.playlist_result(
4265             self._extract_mix_playlist(playlist, playlist_id, data, ytcfg),
4266             playlist_id=playlist_id, playlist_title=title)
4267
4268     def _extract_availability(self, data):
4269         """
4270         Gets the availability of a given playlist/tab.
4271         Note: Unless YouTube tells us explicitly, we do not assume it is public
4272         @param data: response
4273         """
4274         is_private = is_unlisted = None
4275         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4276         badge_labels = self._extract_badges(renderer)
4277
4278         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4279         privacy_dropdown_entries = try_get(
4280             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4281         for renderer_dict in privacy_dropdown_entries:
4282             is_selected = try_get(
4283                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4284             if not is_selected:
4285                 continue
4286             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4287             if label:
4288                 badge_labels.add(label.lower())
4289                 break
4290
4291         for badge_label in badge_labels:
4292             if badge_label == 'unlisted':
4293                 is_unlisted = True
4294             elif badge_label == 'private':
4295                 is_private = True
4296             elif badge_label == 'public':
4297                 is_unlisted = is_private = False
4298         return self._availability(is_private, False, False, False, is_unlisted)
4299
4300     @staticmethod
4301     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4302         sidebar_renderer = try_get(
4303             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4304         for item in sidebar_renderer:
4305             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4306             if renderer:
4307                 return renderer
4308
4309     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4310         """
4311         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4312         """
4313         browse_id = params = None
4314         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4315         if not renderer:
4316             return
4317         menu_renderer = try_get(
4318             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4319         for menu_item in menu_renderer:
4320             if not isinstance(menu_item, dict):
4321                 continue
4322             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4323             text = try_get(
4324                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4325             if not text or text.lower() != 'show unavailable videos':
4326                 continue
4327             browse_endpoint = try_get(
4328                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4329             browse_id = browse_endpoint.get('browseId')
4330             params = browse_endpoint.get('params')
4331             break
4332
4333         headers = self.generate_api_headers(
4334             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4335             visitor_data=self._extract_visitor_data(data, ytcfg))
4336         query = {
4337             'params': params or 'wgYCCAA=',
4338             'browseId': browse_id or 'VL%s' % item_id
4339         }
4340         return self._extract_response(
4341             item_id=item_id, headers=headers, query=query,
4342             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4343             note='Downloading API JSON with unavailable videos')
4344
4345     @property
4346     def skip_webpage(self):
4347         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
4348
4349     def _extract_webpage(self, url, item_id, fatal=True):
4350         retries = self.get_param('extractor_retries', 3)
4351         count = -1
4352         webpage = data = last_error = None
4353         while count < retries:
4354             count += 1
4355             # Sometimes youtube returns a webpage with incomplete ytInitialData
4356             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4357             if last_error:
4358                 self.report_warning('%s. Retrying ...' % last_error)
4359             try:
4360                 webpage = self._download_webpage(
4361                     url, item_id,
4362                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4363                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4364             except ExtractorError as e:
4365                 if isinstance(e.cause, network_exceptions):
4366                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4367                         last_error = error_to_compat_str(e.cause or e.msg)
4368                         if count < retries:
4369                             continue
4370                 if fatal:
4371                     raise
4372                 self.report_warning(error_to_compat_str(e))
4373                 break
4374             else:
4375                 try:
4376                     self._extract_and_report_alerts(data)
4377                 except ExtractorError as e:
4378                     if fatal:
4379                         raise
4380                     self.report_warning(error_to_compat_str(e))
4381                     break
4382
4383                 if dict_get(data, ('contents', 'currentVideoEndpoint', 'onResponseReceivedActions')):
4384                     break
4385
4386                 last_error = 'Incomplete yt initial data received'
4387                 if count >= retries:
4388                     if fatal:
4389                         raise ExtractorError(last_error)
4390                     self.report_warning(last_error)
4391                     break
4392
4393         return webpage, data
4394
4395     def _report_playlist_authcheck(self, ytcfg, fatal=True):
4396         """Use if failed to extract ytcfg (and data) from initial webpage"""
4397         if not ytcfg and self.is_authenticated:
4398             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
4399             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
4400                 raise ExtractorError(
4401                     f'{msg}. If you are not downloading private content, or '
4402                     'your cookies are only for the first account and channel,'
4403                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4404                     expected=True)
4405             self.report_warning(msg, only_once=True)
4406
4407     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4408         data = None
4409         if not self.skip_webpage:
4410             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4411             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4412             # Reject webpage data if redirected to home page without explicitly requesting
4413             selected_tab = self._extract_selected_tab(traverse_obj(
4414                 data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list, default=[]), fatal=False) or {}
4415             if (url != 'https://www.youtube.com/feed/recommended'
4416                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
4417                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
4418                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
4419                 if fatal:
4420                     raise ExtractorError(msg, expected=True)
4421                 self.report_warning(msg, only_once=True)
4422         if not data:
4423             self._report_playlist_authcheck(ytcfg, fatal=fatal)
4424             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4425         return data, ytcfg
4426
4427     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4428         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4429         resolve_response = self._extract_response(
4430             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4431             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4432         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4433         for ep_key, ep in endpoints.items():
4434             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4435             if params:
4436                 return self._extract_response(
4437                     item_id=item_id, query=params, ep=ep, headers=headers,
4438                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4439                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
4440         err_note = 'Failed to resolve url (does the playlist exist?)'
4441         if fatal:
4442             raise ExtractorError(err_note, expected=True)
4443         self.report_warning(err_note, item_id)
4444
4445     _SEARCH_PARAMS = None
4446
4447     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
4448         data = {'query': query}
4449         if params is NO_DEFAULT:
4450             params = self._SEARCH_PARAMS
4451         if params:
4452             data['params'] = params
4453
4454         content_keys = (
4455             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
4456             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
4457             # ytmusic search
4458             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
4459             ('continuationContents', ),
4460         )
4461         display_id = f'query "{query}"'
4462         check_get_keys = tuple({keys[0] for keys in content_keys})
4463         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
4464         self._report_playlist_authcheck(ytcfg, fatal=False)
4465
4466         continuation_list = [None]
4467         search = None
4468         for page_num in itertools.count(1):
4469             data.update(continuation_list[0] or {})
4470             headers = self.generate_api_headers(
4471                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
4472             search = self._extract_response(
4473                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
4474                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
4475             slr_contents = traverse_obj(search, *content_keys)
4476             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
4477             if not continuation_list[0]:
4478                 break
4479
4480
4481 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4482     IE_DESC = 'YouTube Tabs'
4483     _VALID_URL = r'''(?x:
4484         https?://
4485             (?:\w+\.)?
4486             (?:
4487                 youtube(?:kids)?\.com|
4488                 %(invidious)s
4489             )/
4490             (?:
4491                 (?P<channel_type>channel|c|user|browse)/|
4492                 (?P<not_channel>
4493                     feed/|hashtag/|
4494                     (?:playlist|watch)\?.*?\blist=
4495                 )|
4496                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4497             )
4498             (?P<id>[^/?\#&]+)
4499     )''' % {
4500         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4501         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4502     }
4503     IE_NAME = 'youtube:tab'
4504
4505     _TESTS = [{
4506         'note': 'playlists, multipage',
4507         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4508         'playlist_mincount': 94,
4509         'info_dict': {
4510             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4511             'title': 'Igor Kleiner - Playlists',
4512             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4513             'uploader': 'Igor Kleiner',
4514             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4515             'channel': 'Igor Kleiner',
4516             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4517             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4518             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4519             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4520             'channel_follower_count': int
4521         },
4522     }, {
4523         'note': 'playlists, multipage, different order',
4524         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4525         'playlist_mincount': 94,
4526         'info_dict': {
4527             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4528             'title': 'Igor Kleiner - Playlists',
4529             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4530             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4531             'uploader': 'Igor Kleiner',
4532             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4533             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4534             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4535             'channel': 'Igor Kleiner',
4536             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4537             'channel_follower_count': int
4538         },
4539     }, {
4540         'note': 'playlists, series',
4541         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4542         'playlist_mincount': 5,
4543         'info_dict': {
4544             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4545             'title': '3Blue1Brown - Playlists',
4546             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4547             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4548             'uploader': '3Blue1Brown',
4549             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4550             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4551             'channel': '3Blue1Brown',
4552             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4553             'tags': ['Mathematics'],
4554             'channel_follower_count': int
4555         },
4556     }, {
4557         'note': 'playlists, singlepage',
4558         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4559         'playlist_mincount': 4,
4560         'info_dict': {
4561             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4562             'title': 'ThirstForScience - Playlists',
4563             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4564             'uploader': 'ThirstForScience',
4565             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4566             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4567             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4568             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4569             'tags': 'count:13',
4570             'channel': 'ThirstForScience',
4571             'channel_follower_count': int
4572         }
4573     }, {
4574         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4575         'only_matching': True,
4576     }, {
4577         'note': 'basic, single video playlist',
4578         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4579         'info_dict': {
4580             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4581             'uploader': 'Sergey M.',
4582             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4583             'title': 'youtube-dl public playlist',
4584             'description': '',
4585             'tags': [],
4586             'view_count': int,
4587             'modified_date': '20201130',
4588             'channel': 'Sergey M.',
4589             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4590             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4591             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4592         },
4593         'playlist_count': 1,
4594     }, {
4595         'note': 'empty playlist',
4596         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4597         'info_dict': {
4598             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4599             'uploader': 'Sergey M.',
4600             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4601             'title': 'youtube-dl empty playlist',
4602             'tags': [],
4603             'channel': 'Sergey M.',
4604             'description': '',
4605             'modified_date': '20160902',
4606             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4607             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4608             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4609         },
4610         'playlist_count': 0,
4611     }, {
4612         'note': 'Home tab',
4613         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4614         'info_dict': {
4615             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4616             'title': 'lex will - Home',
4617             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4618             'uploader': 'lex will',
4619             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4620             'channel': 'lex will',
4621             'tags': ['bible', 'history', 'prophesy'],
4622             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4623             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4624             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4625             'channel_follower_count': int
4626         },
4627         'playlist_mincount': 2,
4628     }, {
4629         'note': 'Videos tab',
4630         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4631         'info_dict': {
4632             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4633             'title': 'lex will - Videos',
4634             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4635             'uploader': 'lex will',
4636             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4637             'tags': ['bible', 'history', 'prophesy'],
4638             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4639             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4640             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4641             'channel': 'lex will',
4642             'channel_follower_count': int
4643         },
4644         'playlist_mincount': 975,
4645     }, {
4646         'note': 'Videos tab, sorted by popular',
4647         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4648         'info_dict': {
4649             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4650             'title': 'lex will - Videos',
4651             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4652             'uploader': 'lex will',
4653             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4654             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4655             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4656             'channel': 'lex will',
4657             'tags': ['bible', 'history', 'prophesy'],
4658             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4659             'channel_follower_count': int
4660         },
4661         'playlist_mincount': 199,
4662     }, {
4663         'note': 'Playlists tab',
4664         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4665         'info_dict': {
4666             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4667             'title': 'lex will - Playlists',
4668             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4669             'uploader': 'lex will',
4670             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4671             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4672             'channel': 'lex will',
4673             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4674             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4675             'tags': ['bible', 'history', 'prophesy'],
4676             'channel_follower_count': int
4677         },
4678         'playlist_mincount': 17,
4679     }, {
4680         'note': 'Community tab',
4681         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4682         'info_dict': {
4683             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4684             'title': 'lex will - Community',
4685             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4686             'uploader': 'lex will',
4687             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4688             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4689             'channel': 'lex will',
4690             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4691             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4692             'tags': ['bible', 'history', 'prophesy'],
4693             'channel_follower_count': int
4694         },
4695         'playlist_mincount': 18,
4696     }, {
4697         'note': 'Channels tab',
4698         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4699         'info_dict': {
4700             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4701             'title': 'lex will - Channels',
4702             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4703             'uploader': 'lex will',
4704             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4705             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4706             'channel': 'lex will',
4707             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4708             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4709             'tags': ['bible', 'history', 'prophesy'],
4710             'channel_follower_count': int
4711         },
4712         'playlist_mincount': 12,
4713     }, {
4714         'note': 'Search tab',
4715         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4716         'playlist_mincount': 40,
4717         'info_dict': {
4718             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4719             'title': '3Blue1Brown - Search - linear algebra',
4720             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4721             'uploader': '3Blue1Brown',
4722             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4723             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4724             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4725             'tags': ['Mathematics'],
4726             'channel': '3Blue1Brown',
4727             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4728             'channel_follower_count': int
4729         },
4730     }, {
4731         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4732         'only_matching': True,
4733     }, {
4734         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4735         'only_matching': True,
4736     }, {
4737         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4738         'only_matching': True,
4739     }, {
4740         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4741         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4742         'info_dict': {
4743             'title': '29C3: Not my department',
4744             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4745             'uploader': 'Christiaan008',
4746             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4747             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4748             'tags': [],
4749             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4750             'view_count': int,
4751             'modified_date': '20150605',
4752             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4753             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4754             'channel': 'Christiaan008',
4755         },
4756         'playlist_count': 96,
4757     }, {
4758         'note': 'Large playlist',
4759         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4760         'info_dict': {
4761             'title': 'Uploads from Cauchemar',
4762             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4763             'uploader': 'Cauchemar',
4764             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4765             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4766             'tags': [],
4767             'modified_date': r're:\d{8}',
4768             'channel': 'Cauchemar',
4769             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4770             'view_count': int,
4771             'description': '',
4772             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4773         },
4774         'playlist_mincount': 1123,
4775         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4776     }, {
4777         'note': 'even larger playlist, 8832 videos',
4778         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4779         'only_matching': True,
4780     }, {
4781         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4782         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4783         'info_dict': {
4784             'title': 'Uploads from Interstellar Movie',
4785             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4786             'uploader': 'Interstellar Movie',
4787             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4788             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4789             'tags': [],
4790             'view_count': int,
4791             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4792             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4793             'channel': 'Interstellar Movie',
4794             'description': '',
4795             'modified_date': r're:\d{8}',
4796         },
4797         'playlist_mincount': 21,
4798     }, {
4799         'note': 'Playlist with "show unavailable videos" button',
4800         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4801         'info_dict': {
4802             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4803             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4804             'uploader': 'Phim Siêu Nhân Nhật Bản',
4805             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4806             'view_count': int,
4807             'channel': 'Phim Siêu Nhân Nhật Bản',
4808             'tags': [],
4809             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4810             'description': '',
4811             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4812             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4813             'modified_date': r're:\d{8}',
4814         },
4815         'playlist_mincount': 200,
4816         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4817     }, {
4818         'note': 'Playlist with unavailable videos in page 7',
4819         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4820         'info_dict': {
4821             'title': 'Uploads from BlankTV',
4822             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4823             'uploader': 'BlankTV',
4824             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4825             'channel': 'BlankTV',
4826             'channel_url': 'https://www.youtube.com/c/blanktv',
4827             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4828             'view_count': int,
4829             'tags': [],
4830             'uploader_url': 'https://www.youtube.com/c/blanktv',
4831             'modified_date': r're:\d{8}',
4832             'description': '',
4833         },
4834         'playlist_mincount': 1000,
4835         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4836     }, {
4837         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4838         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4839         'info_dict': {
4840             'title': 'Data Analysis with Dr Mike Pound',
4841             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4842             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4843             'uploader': 'Computerphile',
4844             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4845             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4846             'tags': [],
4847             'view_count': int,
4848             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4849             'channel_url': 'https://www.youtube.com/user/Computerphile',
4850             'channel': 'Computerphile',
4851         },
4852         'playlist_mincount': 11,
4853     }, {
4854         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4855         'only_matching': True,
4856     }, {
4857         'note': 'Playlist URL that does not actually serve a playlist',
4858         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4859         'info_dict': {
4860             'id': 'FqZTN594JQw',
4861             'ext': 'webm',
4862             'title': "Smiley's People 01 detective, Adventure Series, Action",
4863             'uploader': 'STREEM',
4864             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4865             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4866             'upload_date': '20150526',
4867             'license': 'Standard YouTube License',
4868             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4869             'categories': ['People & Blogs'],
4870             'tags': list,
4871             'view_count': int,
4872             'like_count': int,
4873         },
4874         'params': {
4875             'skip_download': True,
4876         },
4877         'skip': 'This video is not available.',
4878         'add_ie': [YoutubeIE.ie_key()],
4879     }, {
4880         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
4881         'only_matching': True,
4882     }, {
4883         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
4884         'only_matching': True,
4885     }, {
4886         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
4887         'info_dict': {
4888             'id': 'GgL890LIznQ',  # This will keep changing
4889             'ext': 'mp4',
4890             'title': str,
4891             'uploader': 'Sky News',
4892             'uploader_id': 'skynews',
4893             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
4894             'upload_date': r're:\d{8}',
4895             'description': str,
4896             'categories': ['News & Politics'],
4897             'tags': list,
4898             'like_count': int,
4899             'release_timestamp': 1642502819,
4900             'channel': 'Sky News',
4901             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
4902             'age_limit': 0,
4903             'view_count': int,
4904             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
4905             'playable_in_embed': True,
4906             'release_date': '20220118',
4907             'availability': 'public',
4908             'live_status': 'is_live',
4909             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
4910             'channel_follower_count': int
4911         },
4912         'params': {
4913             'skip_download': True,
4914         },
4915         'expected_warnings': ['Ignoring subtitle tracks found in '],
4916     }, {
4917         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
4918         'info_dict': {
4919             'id': 'a48o2S1cPoo',
4920             'ext': 'mp4',
4921             'title': 'The Young Turks - Live Main Show',
4922             'uploader': 'The Young Turks',
4923             'uploader_id': 'TheYoungTurks',
4924             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
4925             'upload_date': '20150715',
4926             'license': 'Standard YouTube License',
4927             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
4928             'categories': ['News & Politics'],
4929             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
4930             'like_count': int,
4931         },
4932         'params': {
4933             'skip_download': True,
4934         },
4935         'only_matching': True,
4936     }, {
4937         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
4938         'only_matching': True,
4939     }, {
4940         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
4941         'only_matching': True,
4942     }, {
4943         'note': 'A channel that is not live. Should raise error',
4944         'url': 'https://www.youtube.com/user/numberphile/live',
4945         'only_matching': True,
4946     }, {
4947         'url': 'https://www.youtube.com/feed/trending',
4948         'only_matching': True,
4949     }, {
4950         'url': 'https://www.youtube.com/feed/library',
4951         'only_matching': True,
4952     }, {
4953         'url': 'https://www.youtube.com/feed/history',
4954         'only_matching': True,
4955     }, {
4956         'url': 'https://www.youtube.com/feed/subscriptions',
4957         'only_matching': True,
4958     }, {
4959         'url': 'https://www.youtube.com/feed/watch_later',
4960         'only_matching': True,
4961     }, {
4962         'note': 'Recommended - redirects to home page.',
4963         'url': 'https://www.youtube.com/feed/recommended',
4964         'only_matching': True,
4965     }, {
4966         'note': 'inline playlist with not always working continuations',
4967         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
4968         'only_matching': True,
4969     }, {
4970         'url': 'https://www.youtube.com/course',
4971         'only_matching': True,
4972     }, {
4973         'url': 'https://www.youtube.com/zsecurity',
4974         'only_matching': True,
4975     }, {
4976         'url': 'http://www.youtube.com/NASAgovVideo/videos',
4977         'only_matching': True,
4978     }, {
4979         'url': 'https://www.youtube.com/TheYoungTurks/live',
4980         'only_matching': True,
4981     }, {
4982         'url': 'https://www.youtube.com/hashtag/cctv9',
4983         'info_dict': {
4984             'id': 'cctv9',
4985             'title': '#cctv9',
4986             'tags': [],
4987         },
4988         'playlist_mincount': 350,
4989     }, {
4990         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
4991         'only_matching': True,
4992     }, {
4993         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
4994         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
4995         'only_matching': True
4996     }, {
4997         'note': '/browse/ should redirect to /channel/',
4998         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
4999         'only_matching': True
5000     }, {
5001         'note': 'VLPL, should redirect to playlist?list=PL...',
5002         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5003         'info_dict': {
5004             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5005             'uploader': 'NoCopyrightSounds',
5006             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5007             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5008             'title': 'NCS Releases',
5009             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5010             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5011             'modified_date': r're:\d{8}',
5012             'view_count': int,
5013             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5014             'tags': [],
5015             'channel': 'NoCopyrightSounds',
5016         },
5017         'playlist_mincount': 166,
5018         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5019     }, {
5020         'note': 'Topic, should redirect to playlist?list=UU...',
5021         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5022         'info_dict': {
5023             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5024             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5025             'title': 'Uploads from Royalty Free Music - Topic',
5026             'uploader': 'Royalty Free Music - Topic',
5027             'tags': [],
5028             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5029             'channel': 'Royalty Free Music - Topic',
5030             'view_count': int,
5031             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5032             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5033             'modified_date': r're:\d{8}',
5034             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5035             'description': '',
5036         },
5037         'expected_warnings': [
5038             'The URL does not have a videos tab',
5039             r'[Uu]navailable videos (are|will be) hidden',
5040         ],
5041         'playlist_mincount': 101,
5042     }, {
5043         'note': 'Topic without a UU playlist',
5044         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5045         'info_dict': {
5046             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5047             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5048             'tags': [],
5049         },
5050         'expected_warnings': [
5051             'the playlist redirect gave error',
5052         ],
5053         'playlist_mincount': 9,
5054     }, {
5055         'note': 'Youtube music Album',
5056         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5057         'info_dict': {
5058             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5059             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5060             'tags': [],
5061             'view_count': int,
5062             'description': '',
5063             'availability': 'unlisted',
5064             'modified_date': r're:\d{8}',
5065         },
5066         'playlist_count': 50,
5067     }, {
5068         'note': 'unlisted single video playlist',
5069         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5070         'info_dict': {
5071             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5072             'uploader': 'colethedj',
5073             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5074             'title': 'yt-dlp unlisted playlist test',
5075             'availability': 'unlisted',
5076             'tags': [],
5077             'modified_date': '20211208',
5078             'channel': 'colethedj',
5079             'view_count': int,
5080             'description': '',
5081             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5082             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5083             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5084         },
5085         'playlist_count': 1,
5086     }, {
5087         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5088         'url': 'https://www.youtube.com/feed/recommended',
5089         'info_dict': {
5090             'id': 'recommended',
5091             'title': 'recommended',
5092             'tags': [],
5093         },
5094         'playlist_mincount': 50,
5095         'params': {
5096             'skip_download': True,
5097             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5098         },
5099     }, {
5100         'note': 'API Fallback: /videos tab, sorted by oldest first',
5101         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5102         'info_dict': {
5103             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5104             'title': 'Cody\'sLab - Videos',
5105             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5106             'uploader': 'Cody\'sLab',
5107             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5108             'channel': 'Cody\'sLab',
5109             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5110             'tags': [],
5111             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5112             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5113             'channel_follower_count': int
5114         },
5115         'playlist_mincount': 650,
5116         'params': {
5117             'skip_download': True,
5118             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5119         },
5120     }, {
5121         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5122         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5123         'info_dict': {
5124             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5125             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5126             'title': 'Uploads from Royalty Free Music - Topic',
5127             'uploader': 'Royalty Free Music - Topic',
5128             'modified_date': r're:\d{8}',
5129             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5130             'description': '',
5131             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5132             'tags': [],
5133             'channel': 'Royalty Free Music - Topic',
5134             'view_count': int,
5135             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5136         },
5137         'expected_warnings': [
5138             'does not have a videos tab',
5139             r'[Uu]navailable videos (are|will be) hidden',
5140         ],
5141         'playlist_mincount': 101,
5142         'params': {
5143             'skip_download': True,
5144             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5145         },
5146     }, {
5147         'note': 'non-standard redirect to regional channel',
5148         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5149         'only_matching': True
5150     }, {
5151         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5152         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5153         'info_dict': {
5154             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5155             'modified_date': '20220407',
5156             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5157             'tags': [],
5158             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5159             'uploader': 'pukkandan',
5160             'availability': 'unlisted',
5161             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5162             'channel': 'pukkandan',
5163             'description': 'Test for collaborative playlist',
5164             'title': 'yt-dlp test - collaborative playlist',
5165             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5166         },
5167         'playlist_mincount': 2
5168     }]
5169
5170     @classmethod
5171     def suitable(cls, url):
5172         return False if YoutubeIE.suitable(url) else super().suitable(url)
5173
5174     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
5175
5176     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
5177     def _real_extract(self, url, smuggled_data):
5178         item_id = self._match_id(url)
5179         url = compat_urlparse.urlunparse(
5180             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
5181         compat_opts = self.get_param('compat_opts', [])
5182
5183         def get_mobj(url):
5184             mobj = self._URL_RE.match(url).groupdict()
5185             mobj.update((k, '') for k, v in mobj.items() if v is None)
5186             return mobj
5187
5188         mobj, redirect_warning = get_mobj(url), None
5189         # Youtube returns incomplete data if tabname is not lower case
5190         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
5191         if is_channel:
5192             if smuggled_data.get('is_music_url'):
5193                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
5194                     item_id = item_id[2:]
5195                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
5196                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
5197                     mdata = self._extract_tab_endpoint(
5198                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
5199                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
5200                                         get_all=False, expected_type=compat_str)
5201                     if not murl:
5202                         raise ExtractorError('Failed to resolve album to playlist')
5203                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
5204                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
5205                     pre = f'https://www.youtube.com/channel/{item_id}'
5206
5207         original_tab_name = tab
5208         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
5209             # Home URLs should redirect to /videos/
5210             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
5211                                 'To download only the videos in the home page, add a "/featured" to the URL')
5212             tab = '/videos'
5213
5214         url = ''.join((pre, tab, post))
5215         mobj = get_mobj(url)
5216
5217         # Handle both video/playlist URLs
5218         qs = parse_qs(url)
5219         video_id, playlist_id = (qs.get(key, [None])[0] for key in ('v', 'list'))
5220
5221         if not video_id and mobj['not_channel'].startswith('watch'):
5222             if not playlist_id:
5223                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
5224                 raise ExtractorError('Unable to recognize tab page')
5225             # Common mistake: https://www.youtube.com/watch?list=playlist_id
5226             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
5227             url = f'https://www.youtube.com/playlist?list={playlist_id}'
5228             mobj = get_mobj(url)
5229
5230         if video_id and playlist_id:
5231             if self.get_param('noplaylist'):
5232                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
5233                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5234                                        ie=YoutubeIE.ie_key(), video_id=video_id)
5235             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5236
5237         data, ytcfg = self._extract_data(url, item_id)
5238
5239         # YouTube may provide a non-standard redirect to the regional channel
5240         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
5241         redirect_url = traverse_obj(
5242             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
5243         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
5244             redirect_url = ''.join((
5245                 urljoin('https://www.youtube.com', redirect_url), mobj['tab'], mobj['post']))
5246             self.to_screen(f'This playlist is likely not available in your region. Following redirect to regional playlist {redirect_url}')
5247             return self.url_result(redirect_url, ie=YoutubeTabIE.ie_key())
5248
5249         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5250         if tabs:
5251             selected_tab = self._extract_selected_tab(tabs)
5252             selected_tab_name = selected_tab.get('title', '').lower()
5253             if selected_tab_name == 'home':
5254                 selected_tab_name = 'featured'
5255             requested_tab_name = mobj['tab'][1:]
5256             if 'no-youtube-channel-redirect' not in compat_opts:
5257                 if requested_tab_name == 'live':
5258                     # Live tab should have redirected to the video
5259                     raise ExtractorError('The channel is not currently live', expected=True)
5260                 if requested_tab_name not in ('', selected_tab_name):
5261                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5262                     if not original_tab_name:
5263                         if item_id[:2] == 'UC':
5264                             # Topic channels don't have /videos. Use the equivalent playlist instead
5265                             pl_id = f'UU{item_id[2:]}'
5266                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5267                             try:
5268                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5269                             except ExtractorError:
5270                                 redirect_warning += ' and the playlist redirect gave error'
5271                             else:
5272                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5273                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5274                         if selected_tab_name and selected_tab_name != requested_tab_name:
5275                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5276                     else:
5277                         raise ExtractorError(redirect_warning, expected=True)
5278
5279         if redirect_warning:
5280             self.to_screen(redirect_warning)
5281         self.write_debug(f'Final URL: {url}')
5282
5283         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5284         if 'no-youtube-unavailable-videos' not in compat_opts:
5285             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5286         self._extract_and_report_alerts(data, only_once=True)
5287         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5288         if tabs:
5289             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5290
5291         playlist = traverse_obj(
5292             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5293         if playlist:
5294             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5295
5296         video_id = traverse_obj(
5297             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5298         if video_id:
5299             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5300                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5301             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5302                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5303
5304         raise ExtractorError('Unable to recognize tab page')
5305
5306
5307 class YoutubePlaylistIE(InfoExtractor):
5308     IE_DESC = 'YouTube playlists'
5309     _VALID_URL = r'''(?x)(?:
5310                         (?:https?://)?
5311                         (?:\w+\.)?
5312                         (?:
5313                             (?:
5314                                 youtube(?:kids)?\.com|
5315                                 %(invidious)s
5316                             )
5317                             /.*?\?.*?\blist=
5318                         )?
5319                         (?P<id>%(playlist_id)s)
5320                      )''' % {
5321         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5322         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5323     }
5324     IE_NAME = 'youtube:playlist'
5325     _TESTS = [{
5326         'note': 'issue #673',
5327         'url': 'PLBB231211A4F62143',
5328         'info_dict': {
5329             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5330             'id': 'PLBB231211A4F62143',
5331             'uploader': 'Wickman',
5332             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5333             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5334             'view_count': int,
5335             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5336             'modified_date': r're:\d{8}',
5337             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5338             'channel': 'Wickman',
5339             'tags': [],
5340             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5341         },
5342         'playlist_mincount': 29,
5343     }, {
5344         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5345         'info_dict': {
5346             'title': 'YDL_safe_search',
5347             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5348         },
5349         'playlist_count': 2,
5350         'skip': 'This playlist is private',
5351     }, {
5352         'note': 'embedded',
5353         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5354         'playlist_count': 4,
5355         'info_dict': {
5356             'title': 'JODA15',
5357             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5358             'uploader': 'milan',
5359             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5360             'description': '',
5361             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5362             'tags': [],
5363             'modified_date': '20140919',
5364             'view_count': int,
5365             'channel': 'milan',
5366             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5367             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5368         },
5369         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5370     }, {
5371         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5372         'playlist_mincount': 654,
5373         'info_dict': {
5374             'title': '2018 Chinese New Singles (11/6 updated)',
5375             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5376             'uploader': 'LBK',
5377             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5378             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5379             'channel': 'LBK',
5380             'view_count': int,
5381             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5382             'tags': [],
5383             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5384             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5385             'modified_date': r're:\d{8}',
5386         },
5387         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5388     }, {
5389         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5390         'only_matching': True,
5391     }, {
5392         # music album playlist
5393         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5394         'only_matching': True,
5395     }]
5396
5397     @classmethod
5398     def suitable(cls, url):
5399         if YoutubeTabIE.suitable(url):
5400             return False
5401         from ..utils import parse_qs
5402         qs = parse_qs(url)
5403         if qs.get('v', [None])[0]:
5404             return False
5405         return super().suitable(url)
5406
5407     def _real_extract(self, url):
5408         playlist_id = self._match_id(url)
5409         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5410         url = update_url_query(
5411             'https://www.youtube.com/playlist',
5412             parse_qs(url) or {'list': playlist_id})
5413         if is_music_url:
5414             url = smuggle_url(url, {'is_music_url': True})
5415         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5416
5417
5418 class YoutubeYtBeIE(InfoExtractor):
5419     IE_DESC = 'youtu.be'
5420     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5421     _TESTS = [{
5422         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5423         'info_dict': {
5424             'id': 'yeWKywCrFtk',
5425             'ext': 'mp4',
5426             'title': 'Small Scale Baler and Braiding Rugs',
5427             'uploader': 'Backus-Page House Museum',
5428             'uploader_id': 'backuspagemuseum',
5429             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5430             'upload_date': '20161008',
5431             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5432             'categories': ['Nonprofits & Activism'],
5433             'tags': list,
5434             'like_count': int,
5435             'age_limit': 0,
5436             'playable_in_embed': True,
5437             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5438             'channel': 'Backus-Page House Museum',
5439             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5440             'live_status': 'not_live',
5441             'view_count': int,
5442             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5443             'availability': 'public',
5444             'duration': 59,
5445         },
5446         'params': {
5447             'noplaylist': True,
5448             'skip_download': True,
5449         },
5450     }, {
5451         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5452         'only_matching': True,
5453     }]
5454
5455     def _real_extract(self, url):
5456         mobj = self._match_valid_url(url)
5457         video_id = mobj.group('id')
5458         playlist_id = mobj.group('playlist_id')
5459         return self.url_result(
5460             update_url_query('https://www.youtube.com/watch', {
5461                 'v': video_id,
5462                 'list': playlist_id,
5463                 'feature': 'youtu.be',
5464             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5465
5466
5467 class YoutubeLivestreamEmbedIE(InfoExtractor):
5468     IE_DESC = 'YouTube livestream embeds'
5469     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5470     _TESTS = [{
5471         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5472         'only_matching': True,
5473     }]
5474
5475     def _real_extract(self, url):
5476         channel_id = self._match_id(url)
5477         return self.url_result(
5478             f'https://www.youtube.com/channel/{channel_id}/live',
5479             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5480
5481
5482 class YoutubeYtUserIE(InfoExtractor):
5483     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5484     IE_NAME = 'youtube:user'
5485     _VALID_URL = r'ytuser:(?P<id>.+)'
5486     _TESTS = [{
5487         'url': 'ytuser:phihag',
5488         'only_matching': True,
5489     }]
5490
5491     def _real_extract(self, url):
5492         user_id = self._match_id(url)
5493         return self.url_result(
5494             'https://www.youtube.com/user/%s/videos' % user_id,
5495             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5496
5497
5498 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5499     IE_NAME = 'youtube:favorites'
5500     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5501     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5502     _LOGIN_REQUIRED = True
5503     _TESTS = [{
5504         'url': ':ytfav',
5505         'only_matching': True,
5506     }, {
5507         'url': ':ytfavorites',
5508         'only_matching': True,
5509     }]
5510
5511     def _real_extract(self, url):
5512         return self.url_result(
5513             'https://www.youtube.com/playlist?list=LL',
5514             ie=YoutubeTabIE.ie_key())
5515
5516
5517 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
5518     IE_NAME = 'youtube:notif'
5519     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
5520     _VALID_URL = r':ytnotif(?:ication)?s?'
5521     _LOGIN_REQUIRED = True
5522     _TESTS = [{
5523         'url': ':ytnotif',
5524         'only_matching': True,
5525     }, {
5526         'url': ':ytnotifications',
5527         'only_matching': True,
5528     }]
5529
5530     def _extract_notification_menu(self, response, continuation_list):
5531         notification_list = traverse_obj(
5532             response,
5533             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
5534             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
5535             expected_type=list) or []
5536         continuation_list[0] = None
5537         for item in notification_list:
5538             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
5539             if entry:
5540                 yield entry
5541             continuation = item.get('continuationItemRenderer')
5542             if continuation:
5543                 continuation_list[0] = continuation
5544
5545     def _extract_notification_renderer(self, notification):
5546         video_id = traverse_obj(
5547             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
5548         url = f'https://www.youtube.com/watch?v={video_id}'
5549         channel_id = None
5550         if not video_id:
5551             browse_ep = traverse_obj(
5552                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
5553             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
5554             post_id = self._search_regex(
5555                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
5556                 'post id', default=None)
5557             if not channel_id or not post_id:
5558                 return
5559             # The direct /post url redirects to this in the browser
5560             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
5561
5562         channel = traverse_obj(
5563             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
5564             expected_type=str)
5565         title = self._search_regex(
5566             rf'{re.escape(channel)} [^:]+: (.+)', self._get_text(notification, 'shortMessage'),
5567             'video title', default=None)
5568         if title:
5569             title = title.replace('\xad', '')  # remove soft hyphens
5570         upload_date = (strftime_or_none(self._extract_time_text(notification, 'sentTimeText')[0], '%Y%m%d')
5571                        if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE.ie_key())
5572                        else None)
5573         return {
5574             '_type': 'url',
5575             'url': url,
5576             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
5577             'video_id': video_id,
5578             'title': title,
5579             'channel_id': channel_id,
5580             'channel': channel,
5581             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
5582             'upload_date': upload_date,
5583         }
5584
5585     def _notification_menu_entries(self, ytcfg):
5586         continuation_list = [None]
5587         response = None
5588         for page in itertools.count(1):
5589             ctoken = traverse_obj(
5590                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
5591             response = self._extract_response(
5592                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
5593                 ep='notification/get_notification_menu', check_get_keys='actions',
5594                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
5595             yield from self._extract_notification_menu(response, continuation_list)
5596             if not continuation_list[0]:
5597                 break
5598
5599     def _real_extract(self, url):
5600         display_id = 'notifications'
5601         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
5602         self._report_playlist_authcheck(ytcfg)
5603         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
5604
5605
5606 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5607     IE_DESC = 'YouTube search'
5608     IE_NAME = 'youtube:search'
5609     _SEARCH_KEY = 'ytsearch'
5610     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5611     _TESTS = [{
5612         'url': 'ytsearch5:youtube-dl test video',
5613         'playlist_count': 5,
5614         'info_dict': {
5615             'id': 'youtube-dl test video',
5616             'title': 'youtube-dl test video',
5617         }
5618     }]
5619
5620
5621 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5622     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5623     _SEARCH_KEY = 'ytsearchdate'
5624     IE_DESC = 'YouTube search, newest videos first'
5625     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5626     _TESTS = [{
5627         'url': 'ytsearchdate5:youtube-dl test video',
5628         'playlist_count': 5,
5629         'info_dict': {
5630             'id': 'youtube-dl test video',
5631             'title': 'youtube-dl test video',
5632         }
5633     }]
5634
5635
5636 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5637     IE_DESC = 'YouTube search URLs with sorting and filter support'
5638     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5639     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5640     _TESTS = [{
5641         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5642         'playlist_mincount': 5,
5643         'info_dict': {
5644             'id': 'youtube-dl test video',
5645             'title': 'youtube-dl test video',
5646         }
5647     }, {
5648         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5649         'playlist_mincount': 5,
5650         'info_dict': {
5651             'id': 'python',
5652             'title': 'python',
5653         }
5654     }, {
5655         'url': 'https://www.youtube.com/results?search_query=%23cats',
5656         'playlist_mincount': 1,
5657         'info_dict': {
5658             'id': '#cats',
5659             'title': '#cats',
5660             'entries': [{
5661                 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
5662                 'title': '#cats',
5663             }],
5664         },
5665     }, {
5666         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5667         'only_matching': True,
5668     }]
5669
5670     def _real_extract(self, url):
5671         qs = parse_qs(url)
5672         query = (qs.get('search_query') or qs.get('q'))[0]
5673         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5674
5675
5676 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
5677     IE_DESC = 'YouTube music search URLs with selectable sections (Eg: #songs)'
5678     IE_NAME = 'youtube:music:search_url'
5679     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
5680     _TESTS = [{
5681         'url': 'https://music.youtube.com/search?q=royalty+free+music',
5682         'playlist_count': 16,
5683         'info_dict': {
5684             'id': 'royalty free music',
5685             'title': 'royalty free music',
5686         }
5687     }, {
5688         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
5689         'playlist_mincount': 30,
5690         'info_dict': {
5691             'id': 'royalty free music - songs',
5692             'title': 'royalty free music - songs',
5693         },
5694         'params': {'extract_flat': 'in_playlist'}
5695     }, {
5696         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
5697         'playlist_mincount': 30,
5698         'info_dict': {
5699             'id': 'royalty free music - community playlists',
5700             'title': 'royalty free music - community playlists',
5701         },
5702         'params': {'extract_flat': 'in_playlist'}
5703     }]
5704
5705     _SECTIONS = {
5706         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
5707         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
5708         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
5709         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
5710         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
5711         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
5712     }
5713
5714     def _real_extract(self, url):
5715         qs = parse_qs(url)
5716         query = (qs.get('search_query') or qs.get('q'))[0]
5717         params = qs.get('sp', (None,))[0]
5718         if params:
5719             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
5720         else:
5721             section = compat_urllib_parse_unquote_plus((url.split('#') + [''])[1]).lower()
5722             params = self._SECTIONS.get(section)
5723             if not params:
5724                 section = None
5725         title = join_nonempty(query, section, delim=' - ')
5726         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
5727
5728
5729 class YoutubeFeedsInfoExtractor(InfoExtractor):
5730     """
5731     Base class for feed extractors
5732     Subclasses must define the _FEED_NAME property.
5733     """
5734     _LOGIN_REQUIRED = True
5735
5736     def _real_initialize(self):
5737         YoutubeBaseInfoExtractor._check_login_required(self)
5738
5739     @property
5740     def IE_NAME(self):
5741         return 'youtube:%s' % self._FEED_NAME
5742
5743     def _real_extract(self, url):
5744         return self.url_result(
5745             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
5746
5747
5748 class YoutubeWatchLaterIE(InfoExtractor):
5749     IE_NAME = 'youtube:watchlater'
5750     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5751     _VALID_URL = r':ytwatchlater'
5752     _TESTS = [{
5753         'url': ':ytwatchlater',
5754         'only_matching': True,
5755     }]
5756
5757     def _real_extract(self, url):
5758         return self.url_result(
5759             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5760
5761
5762 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5763     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5764     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5765     _FEED_NAME = 'recommended'
5766     _LOGIN_REQUIRED = False
5767     _TESTS = [{
5768         'url': ':ytrec',
5769         'only_matching': True,
5770     }, {
5771         'url': ':ytrecommended',
5772         'only_matching': True,
5773     }, {
5774         'url': 'https://youtube.com',
5775         'only_matching': True,
5776     }]
5777
5778
5779 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5780     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5781     _VALID_URL = r':ytsub(?:scription)?s?'
5782     _FEED_NAME = 'subscriptions'
5783     _TESTS = [{
5784         'url': ':ytsubs',
5785         'only_matching': True,
5786     }, {
5787         'url': ':ytsubscriptions',
5788         'only_matching': True,
5789     }]
5790
5791
5792 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5793     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5794     _VALID_URL = r':ythis(?:tory)?'
5795     _FEED_NAME = 'history'
5796     _TESTS = [{
5797         'url': ':ythistory',
5798         'only_matching': True,
5799     }]
5800
5801
5802 class YoutubeTruncatedURLIE(InfoExtractor):
5803     IE_NAME = 'youtube:truncated_url'
5804     IE_DESC = False  # Do not list
5805     _VALID_URL = r'''(?x)
5806         (?:https?://)?
5807         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5808         (?:watch\?(?:
5809             feature=[a-z_]+|
5810             annotation_id=annotation_[^&]+|
5811             x-yt-cl=[0-9]+|
5812             hl=[^&]*|
5813             t=[0-9]+
5814         )?
5815         |
5816             attribution_link\?a=[^&]+
5817         )
5818         $
5819     '''
5820
5821     _TESTS = [{
5822         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5823         'only_matching': True,
5824     }, {
5825         'url': 'https://www.youtube.com/watch?',
5826         'only_matching': True,
5827     }, {
5828         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5829         'only_matching': True,
5830     }, {
5831         'url': 'https://www.youtube.com/watch?feature=foo',
5832         'only_matching': True,
5833     }, {
5834         'url': 'https://www.youtube.com/watch?hl=en-GB',
5835         'only_matching': True,
5836     }, {
5837         'url': 'https://www.youtube.com/watch?t=2372',
5838         'only_matching': True,
5839     }]
5840
5841     def _real_extract(self, url):
5842         raise ExtractorError(
5843             'Did you forget to quote the URL? Remember that & is a meta '
5844             'character in most shells, so you want to put the URL in quotes, '
5845             'like  youtube-dl '
5846             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5847             ' or simply  youtube-dl BaW_jenozKc  .',
5848             expected=True)
5849
5850
5851 class YoutubeClipIE(InfoExtractor):
5852     IE_NAME = 'youtube:clip'
5853     IE_DESC = False  # Do not list
5854     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/'
5855
5856     def _real_extract(self, url):
5857         self.report_warning('YouTube clips are not currently supported. The entire video will be downloaded instead')
5858         return self.url_result(url, 'Generic')
5859
5860
5861 class YoutubeTruncatedIDIE(InfoExtractor):
5862     IE_NAME = 'youtube:truncated_id'
5863     IE_DESC = False  # Do not list
5864     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
5865
5866     _TESTS = [{
5867         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
5868         'only_matching': True,
5869     }]
5870
5871     def _real_extract(self, url):
5872         video_id = self._match_id(url)
5873         raise ExtractorError(
5874             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
5875             expected=True)